Simplification to the fix in the prior check-in.

FossilOrigin-Name: f79cb748c9447ca4d2184a24219feb19571eb79870e0bedf14704743a3d5f8b9
The sqlite3ExprCompare() routine should always compare the same variables
2024-11-28 02:09:27 +00:00 · 2024-11-28 01:47:44 +00:00 · 2024-11-28 00:58:37 +00:00 · 2024-11-28 00:34:20 +00:00 · 2024-11-27 18:01:25 +00:00 · 2024-11-26 23:40:54 +00:00
50 changed files with 1849 additions and 674 deletions
@@ -9,7 +9,6 @@
 # The docs for many of its variables are in the primary static
 # makefile, main.mk (which this one includes at runtime).
 #
-.POSIX: #maintenance reminder: X:=Y is not POSIX-portable
 all:
 ########################################################################
 #
@@ -144,7 +143,7 @@ LDFLAGS.configure = @LDFLAGS@
 # CFLAGS.core is documented in main.mk.
 #
 CFLAGS.core = @SH_CFLAGS@
-LDFLAGS.shobj = @SHOBJ_LDFLAGS@
+LDFLAGS.shlib = @SH_LDFLAGS@
 LDFLAGS.zlib = @LDFLAGS_ZLIB@
 LDFLAGS.math = @LDFLAGS_MATH@
 LDFLAGS.rpath = @LDFLAGS_RPATH@
@@ -155,6 +154,7 @@ CFLAGS.readline = @CFLAGS_READLINE@
 LDFLAGS.icu = @LDFLAGS_ICU@
 CFLAGS.icu = @CFLAGS_ICU@
 LDFLAGS.libsqlite3.soname = @LDFLAGS_LIBSQLITE3_SONAME@
+# soname: see https://sqlite.org/src/forumpost/5a3b44f510df8ded
 ENABLE_SHARED = @ENABLE_SHARED@
 ENABLE_STATIC = @ENABLE_STATIC@
 HAVE_WASI_SDK = @HAVE_WASI_SDK@
@@ -184,11 +184,6 @@ JIMSH = ./jimsh$(TEXE)
 B.tclsh = @BTCLSH@
 $(B.tclsh):

-#
-# $(CFLAGS.libsqlite3) is documented in main.mk.
-#
-CFLAGS.libsqlite3 = -DSQLITE_TEMP_STORE=@TEMP_STORE@
-
 #
 # $(OPT_FEATURE_FLAGS) is documented in main.mk.
 #
@@ -288,6 +283,7 @@ AS_AUTO_DEF = $(TOP)/auto.def
 AS_AUTORECONFIG = @SQLITE_AUTORECONFIG@

 USE_AMALGAMATION ?= @USE_AMALGAMATION@
+LINK_TOOLS_DYNAMICALLY ?= @LINK_TOOLS_DYNAMICALLY@
 AMALGAMATION_GEN_FLAGS ?= --linemacros=@AMALGAMATION_LINE_MACROS@

 #
@@ -36,7 +36,8 @@ set DUMP_DEFINES_JSON ""; #./config.defines.json
 #
 # 1) --debug is used by autosetup itself, so we have to rename it to
 # --with-debug. We cannot use --enable-debug because that is, for
-# autosetup, and alias for --debug=1.
+# autosetup, and alias for --debug=1. Alternately, we can patch
+# autosetup to use --autosetup-debug for its own purposes instead.
 #
 # 2) In autosetup, all flags starting with (--enable, --disable) are
 # forced to be booleans and receive special handling in how they're
@@ -122,8 +123,11 @@ set DUMP_DEFINES_JSON ""; #./config.defines.json
 #
 ########################################################################
 set flags {
+  # When writing {help text blocks}, be aware that autosetup formats
+  # them differently (left-aligned, under the --flag, if the block
+  # starts with a newline.
+
  # <build-modes>
-  with-debug:=1        => {Enable debug build flags}
  shared=1             => {Disable build of shared libary}
  static=1             => {Disable build of static library (mostly)}
  amalgamation=1       => {Disable the amalgamation and instead build all files separately.}
@@ -147,20 +151,19 @@ set flags {
  all                  => {Enable FTS4, FTS5, Geopoly, RTree, Sessions}
  # </lib-feature>
  # <tcl>
-  # --with-tcl=DIR may be either a dir containing tclConfig.sh or a
-  # dir one level up from that from which we can derive a dir
-  # containing tclConfig.sh.
-  with-tcl:            => {Root of path containing tclConfig.sh}
-  # If --with-tclsh=X given, it is used for (A) trying to find
-  # tclConfig.sh and (B) all TCL-based code generation.  Warning: if
-  # its containing dir has multiple tclsh versions, it may select the
-  # wrong tclConfig.sh!
-  with-tclsh:PATH      => {Full pathname of tclsh to use}
-  # --disable-tcl disables building of components which require TCL,
-  # including tests. This tree requires TCL for code generation but
-  # can use the in-tree copy of autosetup/jimsh0.c for that. The
-  # SQLite TCL extension and the test code require a canonical tclsh.
-  tcl=1                => {Disable components which require TCL}
+  with-tcl:DIR         =>
+    {Directory containing tclConfig.sh or a directory one level up from
+     that, from which we can derive a directory containing tclConfig.sh.}
+  with-tclsh:PATH      =>
+    {Full pathname of tclsh to use.  It is used for (A) trying to find
+     tclConfig.sh and (B) all TCL-based code generation.  Warning: if
+     its containing dir has multiple tclsh versions, it may select the
+     wrong tclConfig.sh!}
+  tcl=1                =>
+    {Disable components which require TCL, including all tests.
+     This tree requires TCL for code generation but can use the in-tree
+     copy of autosetup/jimsh0.c for that. The SQLite TCL extension and the
+     test code require a canonical tclsh.}
  # <tcl>
  # <line-editing>
  readline=1           => {Disable readline support}
@@ -193,12 +196,30 @@ set flags {
  with-emsdk:=auto     => {Top-most dir of the Emscripten SDK installation. Default = EMSDK env var.}
  # </alternative-builds>
  # <developer>
+  # Note that using the --debug/--enable-debug flag here requires patching
+  # autosetup/autosetup to rename the --debug to --autosetup-debug.
+  with-debug=0
+  debug=0              =>
+    {Enable debug build flags. This option will impact performance by
+     as much as 4x, as it includes large numbers of assert()s in
+     performance-critical loops.  Never use --debug for production
+     builds.}
+  scanstatus           => {Enable the SQLITE_ENABLE_STMT_SCANSTATUS feature flag}
+  dev                  => {Enable dev-mode build: automatically enables certain other flags}
  test-status          => {Enable status of tests}
  gcov=0               => {Enable coverage testing using gcov}
  linemacros           => {Enable #line macros in the amalgamation}
-  dev                  => {Enable dev-mode build: automatically enables certain other flags}
  dump-defines=0       => {Dump autosetup defines to $DUMP_DEFINES_TXT (for build debugging)}
-  soname:=legacy       => {SONAME for libsqlite3.so. Must be one of: none, auto, legacy}
+  dynlink-tools        => {Dynamically link libsqlite3 to certain tools which normally statically embed it.}
+  soname:=legacy       =>
+    {SONAME for libsqlite3.so. "none", or not using this flag, sets no
+     soname. "legacy" sets it to its historical value of
+     libsqlite3.so.0.  A value matching the glob "libsqlite3.*" sets
+     it to that literal value. Any other value is assumed to be a
+     suffix which gets applied to "libsqlite3.so.",
+     e.g. --soname=9.10 equates to "libsqlite3.so.9.10".
+    }
+  # --soname has a long story behind it: https://sqlite.org/src/forumpost/5a3b44f510df8ded
  # </developer>
 }
 if {"" ne $DUMP_DEFINES_JSON} {
@@ -217,6 +238,7 @@ unset flags
 proj-xfer-options-aliases {
  with-readline-inc => with-readline-cflags
  with-readline-lib => with-readline-ldflags
+  with-debug => debug
 }

 set srcdir $::autosetup(srcdir)
@@ -322,11 +344,13 @@ proj-if-opt-truthy dev {
  # which check for these flags will show the user their updated
  # state.
  proj-opt-set all 1
-  proj-opt-set with-debug 1
+  proj-opt-set debug 1
  proj-opt-set amalgamation 0
  define CFLAGS [get-env CFLAGS {-O0 -g}]
 }

+define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools]
+
 ########################################################################
 # Handle --with-wasi-sdk=DIR
 #
@@ -444,44 +468,32 @@ if {[cc-check-includes zlib.h] && [proj-check-function-in-lib deflate z]} {
 proj-check-rpath  ; # Determine proper rpath-handling flag

 ########################################################################
-# It's not yet clear whether we gain anything from setting -soname,
-# but not having it has been a source of anxiety for some users.
-# Setting it to any value other than its historical value of
-# libsqlite3.so.0 may break dynamic linking of clients which initially
-# linked against a legacy build (with its SONAME of libsqlite3.so.0).
-#
-# To be clear: the ABI has not changed between pre-3.48 and post-3.47
-# builds, but version number 0 (pre-3.48) was a historical remnant
-# from libtool which "should" have always been version number 3 but
-# was not, for reasons lost to history.
-#
-# If the goal is to reduce downstream disruption then we need to
-# retain the SONAME libsqlite3.so.0.  If the goal is to "pull the
-# bandaid off" then switching libsqlite3.so.3 is arguably the right
-# thing to do (at the very real risk of causing a fair amount of
-# downstream disruption for package maintainers).
-#
-# See discussion in/around:
-# https://sqlite.org/forum/forumpost/0c6fc6f46b2cb3
+# "soname" for libsqlite3.so. See discussion at:
+# https://sqlite.org/src/forumpost/5a3b44f510df8ded
 apply {{} {
  define LDFLAGS_LIBSQLITE3_SONAME ""
  if {[proj-opt-was-provided soname]} {
-    set soname [opt-val soname]
+    set soname [join [opt-val soname] ""]
  } else {
-    set soname legacy
+    set soname none; # enabling soname breaks linking for the --dynlink-tools feature
  }
  switch -exact -- $soname {
-    none   { return 0 }
-    auto   { set soname libsqlite3.so.3 }
-    legacy { set soname libsqlite3.so.0 }
+    none - "" { return 0 }
+    auto      { set soname libsqlite3.so.3 }
+    legacy    { set soname libsqlite3.so.0 }
    default {
-      proj-fatal "Invalid value for --soname. Use one of (none, auto, legacy)."
+      if {[string match libsqlite3.* $soname]} {
+        # use it as-is
+      } else {
+        # Assume it's a suffix
+        set soname "libsqlite3.so.${soname}"
+      }
    }
  }
  msg-debug "soname=$soname"
  if {[proj-check-soname $soname]} {
    define LDFLAGS_LIBSQLITE3_SONAME [get-define LDFLAGS_SONAME_PREFIX]$soname
-    msg-result "Setting SONAME: [get-define LDFLAGS_LIBSQLITE3_SONAME]"
+    msg-result "Setting SONAME using: [get-define LDFLAGS_LIBSQLITE3_SONAME]"
  } elseif {[proj-opt-was-provided soname]} {
    # --soname was explicitly requested but not available, so fail fatally
    proj-fatal "This environment does not support SONAME."
@@ -490,7 +502,6 @@ apply {{} {
  }
 }}

-
 proj-define-for-opt shared ENABLE_SHARED "Build shared library?"

 if {![proj-define-for-opt static ENABLE_STATIC \
@@ -509,7 +520,7 @@ proj-define-for-opt linemacros AMALGAMATION_LINE_MACROS \
  "Use #line macros in the amalgamation:"

 msg-checking "SQLITE_DEBUG build? "
-proj-if-opt-truthy with-debug {
+proj-if-opt-truthy debug {
  define SQLITE_DEBUG 1
  define TARGET_DEBUG {-g -DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall}
  msg-result yes
@@ -652,7 +663,7 @@ proc sqlite-check-tcl {} {
  # Export a subset of tclConfig.sh to the current TCL-space.  If $cfg
  # is an empty string, this emits empty-string entries for the
  # various options we're interested in.
-  eval [exec "${srcdir}/tool/tclConfigShToTcl.sh" "$cfg"]
+  eval [exec "${srcdir}/tool/tclConfigShToAutoDef.sh" "$cfg"]

  if {"" eq $with_tclsh && $cfg ne ""} {
    # We have tclConfig.sh but no tclsh. Attempt to locate a tclsh
@@ -857,7 +868,7 @@ apply {{} {
    }
  }
  msg-result $ts
-  define TEMP_STORE $tsn
+  sqlite-add-feature-flag -DSQLITE_TEMP_STORE=$tsn
 }}

 ########################################################################
@@ -1282,6 +1293,7 @@ foreach {boolFlag featureFlag ifSetEvalThis} {
      sqlite-add-feature-flag -DSQLITE_ENABLE_MEMSYS3
    }
  }
+  scanstatus     -DSQLITE_ENABLE_STMT_SCANSTATUS {}
 } {
  proj-if-opt-truthy $boolFlag {
    sqlite-add-feature-flag $featureFlag
@@ -14,6 +14,7 @@ build infrastructure. It is not an [Autosetup][] reference.
  - Symbolic Names of Feature Flags
  - Do Not Update Global Shared State
 - [Updating Autosetup](#updating)
+  - [Patching Autosetup for Project-local changes](#patching)

 ------------------------------------------------------------------------

@@ -304,8 +305,22 @@ $ fossil status # show the modified files
 ```

 Unless the upgrade made any incompatible changes (which is exceedingly
-rare), that's all there is to it. Check over the diff, test the
-configure process, and check it in.
+rare), that's all there is to it.  Then **apply a patch for the change
+described in the following section**, test the configure process, and
+check it in.
+
+<a name="patching"></a>
+Patching Autosetup for Project-local Changes
+------------------------------------------------------------------------
+
+Autosetup reserves the flag name **`--debug`** for its own purposes,
+and its own special handling of `--enable-...` flags makes `--debug`
+an alias for `--enable-debug`. As we have a long history of using
+`--enable-debug` for this project's own purposes, we patch autosetup
+to use the name `--autosetup-debug` in place of `--debug`. That
+requires (as of this writing) four small edits in
+[](/file/autosetup/autosetup), as demonstrated in [check-in
+3296c8d3](/info/3296c8d3).



@@ -9,7 +9,7 @@ dir=`dirname "$0"`; exec "`$dir/autosetup-find-tclsh`" "$0" "$@"
 set autosetup(version) 0.7.2

 # Can be set to 1 to debug early-init problems
-set autosetup(debug) [expr {"--debug" in $argv}]
+set autosetup(debug) [expr {"--autosetup-debug" in $argv}]

 ##################################################################
 #
@@ -98,7 +98,7 @@ proc main {argv} {
 		version         => "display the version of autosetup"
 		ref:=text manual:=text
 		reference:=text => "display the autosetup command reference. 'text', 'wiki', 'asciidoc' or 'markdown'"
-		debug           => "display debugging output as autosetup runs"
+		autosetup-debug => "display debugging output as autosetup runs"
 		install:=.      => "install autosetup to the current or given directory"
 	}
 	if {$autosetup(installed)} {
@@ -132,7 +132,7 @@ proc main {argv} {
 	}

 	# Debugging output (set this early)
-	incr autosetup(debug) [opt-bool debug]
+	incr autosetup(debug) [opt-bool autosetup-debug]
 	incr autosetup(force) [opt-bool force]
 	incr autosetup(msg-quiet) [opt-bool quiet]
 	incr autosetup(msg-timing) [opt-bool timing]
@@ -2530,7 +2530,7 @@ if {[catch {main $argv} msg opts] == 1} {
 	show-notices
 	autosetup-full-error [error-dump $msg $opts $autosetup(debug)]
 	if {!$autosetup(debug)} {
-		puts stderr "Try: '[file tail $autosetup(exe)] --debug' for a full stack trace"
+		puts stderr "Try: '[file tail $autosetup(exe)] --autosetup-debug' for a full stack trace"
 	}
 	exit 1
 }
@@ -85,8 +85,7 @@ proc proj-assert {script {descr ""}} {
  if {1 == [get-env proj-assert 0]} {
    msg-result [proj-bold "asserting: $script"]
  }
-  set x {expr }
-  append x \{ $script \}
+  set x "expr \{ $script \}"
  if {![uplevel 1 $x]} {
    if {"" eq $descr} {
      set descr $script
@@ -312,11 +311,11 @@ proc proj-first-bin-of {args} {
 ########################################################################
 # @proj-opt-was-provided key
 #
-# Returns 1 if the user specifically provided the given configure
-# flag, else 0. This can be used to distinguish between options which
-# have a default value and those which were explicitly provided by the
-# user, even if the latter is done in a way which uses the default
-# value.
+# Returns 1 if the user specifically provided the given configure flag
+# or if it was specifically set using proj-opt-set, else 0. This can
+# be used to distinguish between options which have a default value
+# and those which were explicitly provided by the user, even if the
+# latter is done in a way which uses the default value.
 #
 # For example, with a configure flag defined like:
 #
@@ -1491,7 +1491,7 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
    }else{
      IdxTable *pTab;
      rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
-      if( rc==SQLITE_OK ){
+      if( rc==SQLITE_OK && ALWAYS(pTab!=0) ){
        int i;
        char *zInner = 0;
        char *zOuter = 0;
@@ -319,10 +319,11 @@ static int getNextString(
        Fts3PhraseToken *pToken;

        p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
-        if( !p ) goto no_mem;
-
        zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
-        if( !zTemp ) goto no_mem;
+        if( !zTemp || !p ){
+          rc = SQLITE_NOMEM;
+          goto getnextstring_out;
+        }

        assert( nToken==ii );
        pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
@@ -337,9 +338,6 @@ static int getNextString(
        nToken = ii+1;
      }
    }
-
-    pModule->xClose(pCursor);
-    pCursor = 0;
  }

  if( rc==SQLITE_DONE ){
@@ -347,7 +345,10 @@ static int getNextString(
    char *zBuf = 0;

    p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
-    if( !p ) goto no_mem;
+    if( !p ){
+      rc = SQLITE_NOMEM;
+      goto getnextstring_out;
+    }
    memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
    p->eType = FTSQUERY_PHRASE;
    p->pPhrase = (Fts3Phrase *)&p[1];
@@ -355,11 +356,9 @@ static int getNextString(
    p->pPhrase->nToken = nToken;

    zBuf = (char *)&p->pPhrase->aToken[nToken];
+    assert( nTemp==0 || zTemp );
    if( zTemp ){
      memcpy(zBuf, zTemp, nTemp);
-      sqlite3_free(zTemp);
-    }else{
-      assert( nTemp==0 );
    }

    for(jj=0; jj<p->pPhrase->nToken; jj++){
@@ -369,17 +368,17 @@ static int getNextString(
    rc = SQLITE_OK;
  }

-  *ppExpr = p;
-  return rc;
-no_mem:
-
+ getnextstring_out:
  if( pCursor ){
    pModule->xClose(pCursor);
  }
  sqlite3_free(zTemp);
-  sqlite3_free(p);
-  *ppExpr = 0;
-  return SQLITE_NOMEM;
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(p);
+    p = 0;
+  }
+  *ppExpr = p;
+  return rc;
 }

 /*
@@ -298,16 +298,31 @@ struct Fts5PhraseIter {
 **   value returned by xInstCount(), SQLITE_RANGE is returned.  Otherwise,
 **   output variable (*ppToken) is set to point to a buffer containing the
 **   matching document token, and (*pnToken) to the size of that buffer in 
-**   bytes. This API is not available if the specified token matches a 
-**   prefix query term. In that case both output variables are always set 
-**   to 0.
+**   bytes. 
 **
 **   The output text is not a copy of the document text that was tokenized.
 **   It is the output of the tokenizer module. For tokendata=1 tables, this 
 **   includes any embedded 0x00 and trailing data.
 **
+**   This API may be slow in some cases if the token identified by parameters 
+**   iIdx and iToken matched a prefix token in the query. In most cases, the
+**   first call to this API for each prefix token in the query is forced
+**   to scan the portion of the full-text index that matches the prefix
+**   token to collect the extra data required by this API. If the prefix
+**   token matches a large number of token instances in the document set,
+**   this may be a performance problem. 
+**
+**   If the user knows in advance that a query may use this API for a
+**   prefix token, FTS5 may be configured to collect all required data as part
+**   of the initial querying of the full-text index, avoiding the second scan
+**   entirely. This also causes prefix queries that do not use this API to 
+**   run more slowly and use more memory. FTS5 may be configured in this way
+**   either on a per-table basis using the [FTS5 insttoken | 'insttoken'] 
+**   option, or on a per-query basis using the 
+**   [fts5_insttoken | fts5_insttoken()] user function.
+**
 **   This API can be quite slow if used with an FTS5 table created with the
-**   "detail=none" or "detail=column" option.
+**   "detail=none" or "detail=column" option. 
 **
 ** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
 **   If parameter iCol is less than zero, or greater than or equal to the
@@ -247,7 +247,8 @@ struct Fts5Config {
  char *zRank;                    /* Name of rank function */
  char *zRankArgs;                /* Arguments to rank function */
  int bSecureDelete;              /* 'secure-delete' */
-  int nDeleteMerge;           /* 'deletemerge' */
+  int nDeleteMerge;               /* 'deletemerge' */
+  int bPrefixInsttoken;           /* 'prefix-insttoken' */

  /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
  char **pzErrmsg;
@@ -504,7 +505,14 @@ int sqlite3Fts5StructureTest(Fts5Index*, void*);
 /*
 ** Used by xInstToken():
 */
-int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);
+int sqlite3Fts5IterToken(
+  Fts5IndexIter *pIndexIter, 
+  const char *pToken, int nToken,
+  i64 iRowid,
+  int iCol, 
+  int iOff, 
+  const char **ppOut, int *pnOut
+);

 /*
 ** Insert or remove data to or from the index. Each time a document is 
@@ -1026,6 +1026,19 @@ int sqlite3Fts5ConfigSetValue(
    }else{
      pConfig->bSecureDelete = (bVal ? 1 : 0);
    }
+  }
+
+  else if( 0==sqlite3_stricmp(zKey, "insttoken") ){
+    int bVal = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      bVal = sqlite3_value_int(pVal);
+    }
+    if( bVal<0 ){
+      *pbBadkey = 1;
+    }else{
+      pConfig->bPrefixInsttoken = (bVal ? 1 : 0);
+    }
+
  }else{
    *pbBadkey = 1;
  }
@@ -3046,7 +3046,7 @@ static int fts5ExprPopulatePoslistsCb(
        int rc = sqlite3Fts5PoslistWriterAppend(
            &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
        );
-        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
+        if( rc==SQLITE_OK && (pExpr->pConfig->bTokendata || pT->bPrefix) ){
          int iCol = p->iOff>>32;
          int iTokOff = p->iOff & 0x7FFFFFFF;
          rc = sqlite3Fts5IndexIterWriteTokendata(
@@ -3239,15 +3239,14 @@ int sqlite3Fts5ExprInstToken(
    return SQLITE_RANGE;
  }
  pTerm = &pPhrase->aTerm[iToken];
-  if( pTerm->bPrefix==0 ){
-    if( pExpr->pConfig->bTokendata ){
-      rc = sqlite3Fts5IterToken(
-          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
-      );
-    }else{
-      *ppOut = pTerm->pTerm;
-      *pnOut = pTerm->nFullTerm;
-    }
+  if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){
+    rc = sqlite3Fts5IterToken(
+        pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm, 
+        iRowid, iCol, iOff+iToken, ppOut, pnOut
+    );
+  }else{
+    *ppOut = pTerm->pTerm;
+    *pnOut = pTerm->nFullTerm;
  }
  return rc;
 }
@@ -6203,6 +6203,383 @@ static void fts5MergePrefixLists(
  *p1 = out;
 }

+
+/*
+** Iterate through a range of entries in the FTS index, invoking the xVisit
+** callback for each of them.
+**
+** Parameter pToken points to an nToken buffer containing an FTS index term
+** (i.e. a document term with the preceding 1 byte index identifier -
+** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits
+** all entries for terms that have pToken/nToken as a prefix. If bPrefix
+** is false, then only entries with pToken/nToken as the entire key are
+** visited. 
+**
+** If the current table is a tokendata=1 table, then if bPrefix is true then
+** each index term is treated separately. However, if bPrefix is false, then
+** all index terms corresponding to pToken/nToken are collapsed into a single
+** term before the callback is invoked.
+**
+** The callback invoked for each entry visited is specified by paramter xVisit.
+** Each time it is invoked, it is passed a pointer to the Fts5Index object,
+** a copy of the 7th paramter to this function (pCtx) and a pointer to the
+** iterator that indicates the current entry. If the current entry is the
+** first with a new term (i.e. different from that of the previous entry,
+** including the very first term), then the final two parameters are passed
+** a pointer to the term and its size in bytes, respectively. If the current
+** entry is not the first associated with its term, these two parameters
+** are passed 0.
+**
+** If parameter pColset is not NULL, then it is used to filter entries before
+** the callback is invoked.
+*/
+static int fts5VisitEntries(
+  Fts5Index *p,                   /* Fts5 index object */
+  Fts5Colset *pColset,            /* Columns filter to apply, or NULL */
+  u8 *pToken,                     /* Buffer containing token */
+  int nToken,                     /* Size of buffer pToken in bytes */
+  int bPrefix,                    /* True for a prefix scan */
+  void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int),
+  void *pCtx                      /* Passed as second argument to xVisit() */
+){
+  const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN : 0)
+                  | FTS5INDEX_QUERY_SKIPEMPTY 
+                  | FTS5INDEX_QUERY_NOOUTPUT;
+  Fts5Iter *p1 = 0;     /* Iterator used to gather data from index */
+  int bNewTerm = 1;
+  Fts5Structure *pStruct = fts5StructureRead(p);
+
+  fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1);
+  fts5IterSetOutputCb(&p->rc, p1);
+  for( /* no-op */ ;
+      fts5MultiIterEof(p, p1)==0;
+      fts5MultiIterNext2(p, p1, &bNewTerm)
+  ){
+    Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
+    int nNew = 0;
+    const u8 *pNew = 0;
+
+    p1->xSetOutputs(p1, pSeg);
+    if( p->rc ) break;
+
+    if( bNewTerm ){
+      nNew = pSeg->term.n;
+      pNew = pSeg->term.p;
+      if( nNew<nToken || memcmp(pToken, pNew, nToken) ) break;
+    }
+
+    xVisit(p, pCtx, p1, pNew, nNew);
+  }
+  fts5MultiIterFree(p1);
+
+  fts5StructureRelease(pStruct);
+  return p->rc;
+}
+
+
+/*
+** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an
+** array of these for each row it visits (so all iRowid fields are the same).
+** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an
+** array of these for the entire query (in which case iRowid fields may take
+** a variety of values).
+**
+** Each instance in the array indicates the iterator (and therefore term)
+** associated with position iPos of rowid iRowid. This is used by the
+** xInstToken() API.
+**
+** iRowid:
+**   Rowid for the current entry.
+**
+** iPos:
+**   Position of current entry within row. In the usual ((iCol<<32)+iOff)
+**   format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()).
+**
+** iIter:
+**   If the Fts5TokenDataIter iterator that the entry is part of is
+**   actually an iterator (i.e. with nIter>0, not just a container for
+**   Fts5TokenDataMap structures), then this variable is an index into
+**   the apIter[] array. The corresponding term is that which the iterator
+**   at apIter[iIter] currently points to.
+**
+**   Or, if the Fts5TokenDataIter iterator is just a container object
+**   (nIter==0), then iIter is an index into the term.p[] buffer where
+**   the term is stored.
+**
+** nByte:
+**   In the case where iIter is an index into term.p[], this variable
+**   is the size of the term in bytes. If iIter is an index into apIter[],
+**   this variable is unused.
+*/
+struct Fts5TokenDataMap {
+  i64 iRowid;                     /* Row this token is located in */
+  i64 iPos;                       /* Position of token */
+  int iIter;                      /* Iterator token was read from */
+  int nByte;                      /* Length of token in bytes (or 0) */
+};
+
+/*
+** An object used to supplement Fts5Iter for tokendata=1 iterators.
+**
+** This object serves two purposes. The first is as a container for an array
+** of Fts5TokenDataMap structures, which are used to find the token required
+** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and
+** aMap[] variables.
+*/
+struct Fts5TokenDataIter {
+  int nMapAlloc;                  /* Allocated size of aMap[] in entries */
+  int nMap;                       /* Number of valid entries in aMap[] */
+  Fts5TokenDataMap *aMap;         /* Array of (rowid+pos -> token) mappings */
+
+  /* The following are used for prefix-queries only. */
+  Fts5Buffer terms;
+
+  /* The following are used for other full-token tokendata queries only. */
+  int nIter;
+  int nIterAlloc;
+  Fts5PoslistReader *aPoslistReader;
+  int *aPoslistToIter;
+  Fts5Iter *apIter[1];
+};
+
+/*
+** The two input arrays - a1[] and a2[] - are in sorted order. This function
+** merges the two arrays together and writes the result to output array 
+** aOut[]. aOut[] is guaranteed to be large enough to hold the result.
+**
+** Duplicate entries are copied into the output. So the size of the output
+** array is always (n1+n2) entries.
+*/
+static void fts5TokendataMerge(
+  Fts5TokenDataMap *a1, int n1,   /* Input array 1 */
+  Fts5TokenDataMap *a2, int n2,   /* Input array 2 */
+  Fts5TokenDataMap *aOut          /* Output array */
+){
+  int i1 = 0;
+  int i2 = 0;
+
+  assert( n1>=0 && n2>=0 );
+  while( i1<n1 || i2<n2 ){
+    Fts5TokenDataMap *pOut = &aOut[i1+i2];
+    if( i2>=n2 || (i1<n1 && (
+        a1[i1].iRowid<a2[i2].iRowid
+     || (a1[i1].iRowid==a2[i2].iRowid && a1[i1].iPos<=a2[i2].iPos)
+    ))){
+      memcpy(pOut, &a1[i1], sizeof(Fts5TokenDataMap));
+      i1++;
+    }else{
+      memcpy(pOut, &a2[i2], sizeof(Fts5TokenDataMap));
+      i2++;
+    }
+  }
+}
+
+
+/*
+** Append a mapping to the token-map belonging to object pT.
+*/
+static void fts5TokendataIterAppendMap(
+  Fts5Index *p, 
+  Fts5TokenDataIter *pT, 
+  int iIter,
+  int nByte,
+  i64 iRowid, 
+  i64 iPos
+){
+  if( p->rc==SQLITE_OK ){
+    if( pT->nMap==pT->nMapAlloc ){
+      int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64;
+      int nAlloc = nNew * sizeof(Fts5TokenDataMap);
+      Fts5TokenDataMap *aNew;
+
+      aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc);
+      if( aNew==0 ){
+        p->rc = SQLITE_NOMEM;
+        return;
+      }
+
+      pT->aMap = aNew;
+      pT->nMapAlloc = nNew;
+    }
+
+    pT->aMap[pT->nMap].iRowid = iRowid;
+    pT->aMap[pT->nMap].iPos = iPos;
+    pT->aMap[pT->nMap].iIter = iIter;
+    pT->aMap[pT->nMap].nByte = nByte;
+    pT->nMap++;
+  }
+}
+
+/*
+** Sort the contents of the pT->aMap[] array.
+**
+** The sorting algorithm requries a malloc(). If this fails, an error code
+** is left in Fts5Index.rc before returning.
+*/
+static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){
+  Fts5TokenDataMap *aTmp = 0;
+  int nByte = pT->nMap * sizeof(Fts5TokenDataMap);
+
+  aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte);
+  if( aTmp ){
+    Fts5TokenDataMap *a1 = pT->aMap;
+    Fts5TokenDataMap *a2 = aTmp;
+    i64 nHalf;
+
+    for(nHalf=1; nHalf<pT->nMap; nHalf=nHalf*2){
+      int i1;
+      for(i1=0; i1<pT->nMap; i1+=(nHalf*2)){
+        int n1 = MIN(nHalf, pT->nMap-i1);
+        int n2 = MIN(nHalf, pT->nMap-i1-n1);
+        fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]);
+      }
+      SWAPVAL(Fts5TokenDataMap*, a1, a2);
+    }
+
+    if( a1!=pT->aMap ){
+      memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap));
+    }
+    sqlite3_free(aTmp);
+
+#ifdef SQLITE_DEBUG
+    {
+      int ii;
+      for(ii=1; ii<pT->nMap; ii++){
+        Fts5TokenDataMap *p1 = &pT->aMap[ii-1];
+        Fts5TokenDataMap *p2 = &pT->aMap[ii];
+        assert( p1->iRowid<p2->iRowid 
+             || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos)
+        );
+      }
+    }
+#endif
+  }
+}
+
+/*
+** Delete an Fts5TokenDataIter structure and its contents.
+*/
+static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){
+  if( pSet ){
+    int ii;
+    for(ii=0; ii<pSet->nIter; ii++){
+      fts5MultiIterFree(pSet->apIter[ii]);
+    }
+    fts5BufferFree(&pSet->terms);
+    sqlite3_free(pSet->aPoslistReader);
+    sqlite3_free(pSet->aMap);
+    sqlite3_free(pSet);
+  }
+}
+
+
+/*
+** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata()
+** to pass data to prefixIterSetupTokendataCb().
+*/
+typedef struct TokendataSetupCtx TokendataSetupCtx;
+struct TokendataSetupCtx {
+  Fts5TokenDataIter *pT;          /* Object being populated with mappings */
+  int iTermOff;                   /* Offset of current term in terms.p[] */
+  int nTermByte;                  /* Size of current term in bytes */
+};
+
+/*
+** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This
+** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each
+** position in the current position-list. It doesn't matter that some of
+** these may be out of order - they will be sorted later.
+*/
+static void prefixIterSetupTokendataCb(
+  Fts5Index *p, 
+  void *pCtx, 
+  Fts5Iter *p1, 
+  const u8 *pNew,
+  int nNew
+){
+  TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx;
+  int iPosOff = 0;
+  i64 iPos = 0;
+
+  if( pNew ){
+    pSetup->nTermByte = nNew-1;
+    pSetup->iTermOff = pSetup->pT->terms.n;
+    fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1);
+  }
+
+  while( 0==sqlite3Fts5PoslistNext64(
+     p1->base.pData, p1->base.nData, &iPosOff, &iPos
+  ) ){
+    fts5TokendataIterAppendMap(p, 
+        pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos
+    );
+  }
+}
+
+
+/*
+** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries().
+*/
+typedef struct PrefixSetupCtx PrefixSetupCtx;
+struct PrefixSetupCtx {
+  void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*);
+  void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*);
+  i64 iLastRowid;
+  int nMerge;
+  Fts5Buffer *aBuf;
+  int nBuf;
+  Fts5Buffer doclist;
+  TokendataSetupCtx *pTokendata;
+};
+
+/*
+** fts5VisitEntries() callback used by fts5SetupPrefixIter()
+*/
+static void prefixIterSetupCb(
+  Fts5Index *p, 
+  void *pCtx, 
+  Fts5Iter *p1, 
+  const u8 *pNew,
+  int nNew
+){
+  PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx;
+  const int nMerge = pSetup->nMerge;
+
+  if( p1->base.nData>0 ){
+    if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){
+      int i;
+      for(i=0; p->rc==SQLITE_OK && pSetup->doclist.n; i++){
+        int i1 = i*nMerge;
+        int iStore;
+        assert( i1+nMerge<=pSetup->nBuf );
+        for(iStore=i1; iStore<i1+nMerge; iStore++){
+          if( pSetup->aBuf[iStore].n==0 ){
+            fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]);
+            fts5BufferZero(&pSetup->doclist);
+            break;
+          }
+        }
+        if( iStore==i1+nMerge ){
+          pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]);
+          for(iStore=i1; iStore<i1+nMerge; iStore++){
+            fts5BufferZero(&pSetup->aBuf[iStore]);
+          }
+        }
+      }
+      pSetup->iLastRowid = 0;
+    }
+
+    pSetup->xAppend(
+        p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist
+    );
+    pSetup->iLastRowid = p1->base.iRowid;
+  }
+
+  if( pSetup->pTokendata ){
+    prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew);
+  }
+}
+
 static void fts5SetupPrefixIter(
  Fts5Index *p,                   /* Index to read from */
  int bDesc,                      /* True for "ORDER BY rowid DESC" */
@@ -6213,38 +6590,41 @@ static void fts5SetupPrefixIter(
  Fts5Iter **ppIter               /* OUT: New iterator */
 ){
  Fts5Structure *pStruct;
-  Fts5Buffer *aBuf;
-  int nBuf = 32;
-  int nMerge = 1;
+  PrefixSetupCtx s;
+  TokendataSetupCtx s2;

-  void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*);
-  void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*);
-  if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
-    xMerge = fts5MergeRowidLists;
-    xAppend = fts5AppendRowid;
-  }else{
-    nMerge = FTS5_MERGE_NLIST-1;
-    nBuf = nMerge*8;   /* Sufficient to merge (16^8)==(2^32) lists */
-    xMerge = fts5MergePrefixLists;
-    xAppend = fts5AppendPoslist;
+  memset(&s, 0, sizeof(s));
+  memset(&s2, 0, sizeof(s2));
+
+  s.nMerge = 1;
+  s.iLastRowid = 0;
+  s.nBuf = 32;
+  if( iIdx==0 
+   && p->pConfig->eDetail==FTS5_DETAIL_FULL 
+   && p->pConfig->bPrefixInsttoken 
+  ){
+    s.pTokendata = &s2;
+    s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, sizeof(*s2.pT));
  }

-  aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
+  if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
+    s.xMerge = fts5MergeRowidLists;
+    s.xAppend = fts5AppendRowid;
+  }else{
+    s.nMerge = FTS5_MERGE_NLIST-1;
+    s.nBuf = s.nMerge*8;   /* Sufficient to merge (16^8)==(2^32) lists */
+    s.xMerge = fts5MergePrefixLists;
+    s.xAppend = fts5AppendPoslist;
+  }
+
+  s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf);
  pStruct = fts5StructureRead(p);
-  assert( p->rc!=SQLITE_OK || (aBuf && pStruct) );
+  assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) );

  if( p->rc==SQLITE_OK ){
-    const int flags = FTS5INDEX_QUERY_SCAN 
-                    | FTS5INDEX_QUERY_SKIPEMPTY 
-                    | FTS5INDEX_QUERY_NOOUTPUT;
+    void *pCtx = (void*)&s;
    int i;
-    i64 iLastRowid = 0;
-    Fts5Iter *p1 = 0;     /* Iterator used to gather data from index */
    Fts5Data *pData;
-    Fts5Buffer doclist;
-    int bNewTerm = 1;
-
-    memset(&doclist, 0, sizeof(doclist));

    /* If iIdx is non-zero, then it is the number of a prefix-index for
    ** prefixes 1 character longer than the prefix being queried for. That
@@ -6252,94 +6632,43 @@ static void fts5SetupPrefixIter(
    ** corresponding to the prefix itself. That one is extracted from the
    ** main term index here.  */
    if( iIdx!=0 ){
-      int dummy = 0;
-      const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT;
      pToken[0] = FTS5_MAIN_PREFIX;
-      fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1);
-      fts5IterSetOutputCb(&p->rc, p1);
-      for(;
-        fts5MultiIterEof(p, p1)==0;
-        fts5MultiIterNext2(p, p1, &dummy)
-      ){
-        Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
-        p1->xSetOutputs(p1, pSeg);
-        if( p1->base.nData ){
-          xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist);
-          iLastRowid = p1->base.iRowid;
-        }
-      }
-      fts5MultiIterFree(p1);
+      fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx);
    }

    pToken[0] = FTS5_MAIN_PREFIX + iIdx;
-    fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1);
-    fts5IterSetOutputCb(&p->rc, p1);
+    fts5VisitEntries(p, pColset, pToken, nToken, 1, prefixIterSetupCb, pCtx);

-    for( /* no-op */ ;
-        fts5MultiIterEof(p, p1)==0;
-        fts5MultiIterNext2(p, p1, &bNewTerm)
-    ){
-      Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
-      int nTerm = pSeg->term.n;
-      const u8 *pTerm = pSeg->term.p;
-      p1->xSetOutputs(p1, pSeg);
-
-      assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
-      if( bNewTerm ){
-        if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
-      }
-
-      if( p1->base.nData==0 ) continue;
-      if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){
-        for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
-          int i1 = i*nMerge;
-          int iStore;
-          assert( i1+nMerge<=nBuf );
-          for(iStore=i1; iStore<i1+nMerge; iStore++){
-            if( aBuf[iStore].n==0 ){
-              fts5BufferSwap(&doclist, &aBuf[iStore]);
-              fts5BufferZero(&doclist);
-              break;
-            }
-          }
-          if( iStore==i1+nMerge ){
-            xMerge(p, &doclist, nMerge, &aBuf[i1]);
-            for(iStore=i1; iStore<i1+nMerge; iStore++){
-              fts5BufferZero(&aBuf[iStore]);
-            }
-          }
-        }
-        iLastRowid = 0;
-      }
-
-      xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist);
-      iLastRowid = p1->base.iRowid;
-    }
-
-    assert( (nBuf%nMerge)==0 );
-    for(i=0; i<nBuf; i+=nMerge){
+    assert( (s.nBuf%s.nMerge)==0 );
+    for(i=0; i<s.nBuf; i+=s.nMerge){
      int iFree;
      if( p->rc==SQLITE_OK ){
-        xMerge(p, &doclist, nMerge, &aBuf[i]);
+        s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]);
      }
-      for(iFree=i; iFree<i+nMerge; iFree++){
-        fts5BufferFree(&aBuf[iFree]);
+      for(iFree=i; iFree<i+s.nMerge; iFree++){
+        fts5BufferFree(&s.aBuf[iFree]);
      }
    }
-    fts5MultiIterFree(p1);

-    pData = fts5IdxMalloc(p, sizeof(*pData)+doclist.n+FTS5_DATA_ZERO_PADDING);
+    pData = fts5IdxMalloc(p, sizeof(*pData)+s.doclist.n+FTS5_DATA_ZERO_PADDING);
    if( pData ){
      pData->p = (u8*)&pData[1];
-      pData->nn = pData->szLeaf = doclist.n;
-      if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n);
+      pData->nn = pData->szLeaf = s.doclist.n;
+      if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n);
      fts5MultiIterNew2(p, pData, bDesc, ppIter);
    }
-    fts5BufferFree(&doclist);
+
+    if( p->rc==SQLITE_OK && s.pTokendata ){
+      fts5TokendataIterSortMap(p, s2.pT);
+      (*ppIter)->pTokenDataIter = s2.pT;
+      s2.pT = 0;
+    }
  }

+  fts5TokendataIterDelete(s2.pT);
+  fts5BufferFree(&s.doclist);
  fts5StructureRelease(pStruct);
-  sqlite3_free(aBuf);
+  sqlite3_free(s.aBuf);
 }


@@ -6593,38 +6922,6 @@ static void fts5SegIterSetEOF(Fts5SegIter *pSeg){
  pSeg->pLeaf = 0;
 }

-/*
-** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an
-** array of these for each row it visits. Or, for an iterator used by an
-** "ORDER BY rank" query, it accumulates an array of these for the entire
-** query.
-**
-** Each instance in the array indicates the iterator (and therefore term)
-** associated with position iPos of rowid iRowid. This is used by the
-** xInstToken() API.
-*/
-struct Fts5TokenDataMap {
-  i64 iRowid;                     /* Row this token is located in */
-  i64 iPos;                       /* Position of token */
-  int iIter;                      /* Iterator token was read from */
-};
-
-/*
-** An object used to supplement Fts5Iter for tokendata=1 iterators.
-*/
-struct Fts5TokenDataIter {
-  int nIter;
-  int nIterAlloc;
-
-  int nMap;
-  int nMapAlloc;
-  Fts5TokenDataMap *aMap;
-
-  Fts5PoslistReader *aPoslistReader;
-  int *aPoslistToIter;
-  Fts5Iter *apIter[1];
-};
-
 /*
 ** This function appends iterator pAppend to Fts5TokenDataIter pIn and 
 ** returns the result.
@@ -6661,54 +6958,6 @@ static Fts5TokenDataIter *fts5AppendTokendataIter(
  return pRet;
 }

-/*
-** Delete an Fts5TokenDataIter structure and its contents.
-*/
-static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){
-  if( pSet ){
-    int ii;
-    for(ii=0; ii<pSet->nIter; ii++){
-      fts5MultiIterFree(pSet->apIter[ii]);
-    }
-    sqlite3_free(pSet->aPoslistReader);
-    sqlite3_free(pSet->aMap);
-    sqlite3_free(pSet);
-  }
-}
-
-/*
-** Append a mapping to the token-map belonging to object pT.
-*/
-static void fts5TokendataIterAppendMap(
-  Fts5Index *p, 
-  Fts5TokenDataIter *pT, 
-  int iIter,
-  i64 iRowid, 
-  i64 iPos
-){
-  if( p->rc==SQLITE_OK ){
-    if( pT->nMap==pT->nMapAlloc ){
-      int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64;
-      int nByte = nNew * sizeof(Fts5TokenDataMap);
-      Fts5TokenDataMap *aNew;
-
-      aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte);
-      if( aNew==0 ){
-        p->rc = SQLITE_NOMEM;
-        return;
-      }
-
-      pT->aMap = aNew;
-      pT->nMapAlloc = nNew;
-    }
-
-    pT->aMap[pT->nMap].iRowid = iRowid;
-    pT->aMap[pT->nMap].iPos = iPos;
-    pT->aMap[pT->nMap].iIter = iIter;
-    pT->nMap++;
-  }
-}
-
 /*
 ** The iterator passed as the only argument must be a tokendata=1 iterator
 ** (pIter->pTokenDataIter!=0). This function sets the iterator output
@@ -6749,7 +6998,7 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
    pIter->base.iRowid = iRowid;

    if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){
-      fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, iRowid, -1);
+      fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1);
    }else
    if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){
      int nReader = 0;
@@ -7002,6 +7251,7 @@ static Fts5Iter *fts5SetupTokendataIter(
    pRet = fts5MultiIterAlloc(p, 0);
  }
  if( pRet ){
+    pRet->nSeg = 0;
    pRet->pTokenDataIter = pSet;
    if( pSet ){
      fts5IterSetOutputsTokendata(pRet);
@@ -7017,7 +7267,6 @@ static Fts5Iter *fts5SetupTokendataIter(
  return pRet;
 }

-
 /*
 ** Open a new iterator to iterate though all rowid that match the 
 ** specified token or token prefix.
@@ -7042,6 +7291,11 @@ int sqlite3Fts5IndexQuery(
    int bTokendata = pConfig->bTokendata;
    if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);

+    /* The NOTOKENDATA flag is set when each token in a tokendata=1 table
+    ** should be treated individually, instead of merging all those with
+    ** a common prefix into a single entry. This is used, for example, by
+    ** queries performed as part of an integrity-check, or by the fts5vocab
+    ** module.  */
    if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){
      bTokendata = 0;
    }
@@ -7072,7 +7326,7 @@ int sqlite3Fts5IndexQuery(
    }

    if( bTokendata && iIdx==0 ){
-      buf.p[0] = '0';
+      buf.p[0] = FTS5_MAIN_PREFIX;
      pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset);
    }else if( iIdx<=pConfig->nPrefix ){
      /* Straight index lookup */
@@ -7085,7 +7339,7 @@ int sqlite3Fts5IndexQuery(
        fts5StructureRelease(pStruct);
      }
    }else{
-      /* Scan multiple terms in the main index */
+      /* Scan multiple terms in the main index for a prefix query. */
      int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
      fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet);
      if( pRet==0 ){
@@ -7121,7 +7375,8 @@ int sqlite3Fts5IndexQuery(
 int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){
  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  assert( pIter->pIndex->rc==SQLITE_OK );
-  if( pIter->pTokenDataIter ){
+  if( pIter->nSeg==0 ){
+    assert( pIter->pTokenDataIter );
    fts5TokendataIterNext(pIter, 0, 0);
  }else{
    fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
@@ -7158,7 +7413,8 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){
 */
 int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){
  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  if( pIter->pTokenDataIter ){
+  if( pIter->nSeg==0 ){
+    assert( pIter->pTokenDataIter );
    fts5TokendataIterNext(pIter, 1, iMatch);
  }else{
    fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
@@ -7177,14 +7433,60 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
  return (z ? &z[1] : 0);
 }

+/*
+** pIter is a prefix query. This function populates pIter->pTokenDataIter
+** with an Fts5TokenDataIter object containing mappings for all rows
+** matched by the query.
+*/
+static int fts5SetupPrefixIterTokendata(
+  Fts5Iter *pIter,
+  const char *pToken,             /* Token prefix to search for */
+  int nToken                      /* Size of pToken in bytes */
+){
+  Fts5Index *p = pIter->pIndex;
+  Fts5Buffer token = {0, 0, 0};
+  TokendataSetupCtx ctx;
+
+  memset(&ctx, 0, sizeof(ctx));
+
+  fts5BufferGrow(&p->rc, &token, nToken+1);
+  ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*ctx.pT));
+
+  if( p->rc==SQLITE_OK ){
+
+    /* Fill in the token prefix to search for */
+    token.p[0] = FTS5_MAIN_PREFIX;
+    memcpy(&token.p[1], pToken, nToken);
+    token.n = nToken+1;
+
+    fts5VisitEntries(
+        p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx
+    );
+
+    fts5TokendataIterSortMap(p, ctx.pT);
+  }
+
+  if( p->rc==SQLITE_OK ){
+    pIter->pTokenDataIter = ctx.pT;
+  }else{
+    fts5TokendataIterDelete(ctx.pT);
+  }
+  fts5BufferFree(&token);
+
+  return fts5IndexReturn(p);
+}
+
 /*
 ** This is used by xInstToken() to access the token at offset iOff, column
 ** iCol of row iRowid. The token is returned via output variables *ppOut
 ** and *pnOut. The iterator passed as the first argument must be a tokendata=1
 ** iterator (pIter->pTokenDataIter!=0).
+**
+** pToken/nToken:
 */
 int sqlite3Fts5IterToken(
  Fts5IndexIter *pIndexIter, 
+  const char *pToken, int nToken,
  i64 iRowid,
  int iCol, 
  int iOff, 
@@ -7192,13 +7494,22 @@ int sqlite3Fts5IterToken(
 ){
  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  Fts5TokenDataIter *pT = pIter->pTokenDataIter;
-  Fts5TokenDataMap *aMap = pT->aMap;
  i64 iPos = (((i64)iCol)<<32) + iOff;
-
+  Fts5TokenDataMap *aMap = 0;
  int i1 = 0;
-  int i2 = pT->nMap;
+  int i2 = 0;
  int iTest = 0;

+  assert( pT || (pToken && pIter->nSeg>0) );
+  if( pT==0 ){
+    int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken);
+    if( rc!=SQLITE_OK ) return rc;
+    pT = pIter->pTokenDataIter;
+  }
+
+  i2 = pT->nMap;
+  aMap = pT->aMap;
+
  while( i2>i1 ){
    iTest = (i1 + i2) / 2;

@@ -7221,9 +7532,15 @@ int sqlite3Fts5IterToken(
  }

  if( i2>i1 ){
-    Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
-    *ppOut = (const char*)pMap->aSeg[0].term.p+1;
-    *pnOut = pMap->aSeg[0].term.n-1;
+    if( pIter->nSeg==0 ){
+      Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
+      *ppOut = (const char*)pMap->aSeg[0].term.p+1;
+      *pnOut = pMap->aSeg[0].term.n-1;
+    }else{
+      Fts5TokenDataMap *p = &aMap[iTest];
+      *ppOut = (const char*)&pT->terms.p[p->iIter];
+      *pnOut = aMap[iTest].nByte;
+    }
  }

  return SQLITE_OK;
@@ -7235,7 +7552,9 @@ int sqlite3Fts5IterToken(
 */
 void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){
  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  if( pIter && pIter->pTokenDataIter ){
+  if( pIter && pIter->pTokenDataIter 
+   && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL)
+  ){
    pIter->pTokenDataIter->nMap = 0;
  }
 }
@@ -7255,17 +7574,29 @@ int sqlite3Fts5IndexIterWriteTokendata(
  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  Fts5TokenDataIter *pT = pIter->pTokenDataIter;
  Fts5Index *p = pIter->pIndex;
-  int ii;
+  i64 iPos = (((i64)iCol)<<32) + iOff;

  assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL );
-  assert( pIter->pTokenDataIter );
-
-  for(ii=0; ii<pT->nIter; ii++){
-    Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
-    if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
-  }
-  if( ii<pT->nIter ){
-    fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff);
+  assert( pIter->pTokenDataIter || pIter->nSeg>0 );
+  if( pIter->nSeg>0 ){
+    /* This is a prefix term iterator. */
+    if( pT==0 ){
+      pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT));
+      pIter->pTokenDataIter = pT;
+    }
+    if( pT ){
+      fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos);
+      fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken);
+    }
+  }else{
+    int ii;
+    for(ii=0; ii<pT->nIter; ii++){
+      Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
+      if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
+    }
+    if( ii<pT->nIter ){
+      fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos);
+    }
  }
  return fts5IndexReturn(p);
 }
@@ -93,6 +93,7 @@ struct Fts5Global {
 #define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
 #define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr))

+#define FTS5_INSTTOKEN_SUBTYPE 73

 /*
 ** Each auxiliary function registered with the FTS5 module is represented
@@ -1418,6 +1419,7 @@ static int fts5FilterMethod(
  sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  int iCol;                       /* Column on LHS of MATCH operator */
  char **pzErrmsg = pConfig->pzErrmsg;
+  int bPrefixInsttoken = pConfig->bPrefixInsttoken;
  int i;
  int iIdxStr = 0;
  Fts5Expr *pExpr = 0;
@@ -1453,6 +1455,9 @@ static int fts5FilterMethod(
        rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset);
        if( rc!=SQLITE_OK ) goto filter_out;
        if( zText==0 ) zText = "";
+        if( sqlite3_value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE ){
+          pConfig->bPrefixInsttoken = 1;
+        }

        iCol = 0;
        do{
@@ -1593,6 +1598,7 @@ static int fts5FilterMethod(
 filter_out:
  sqlite3Fts5ExprFree(pExpr);
  pConfig->pzErrmsg = pzErrmsg;
+  pConfig->bPrefixInsttoken = bPrefixInsttoken;
  return rc;
 }

@@ -3652,6 +3658,19 @@ static void fts5LocaleFunc(
  }
 }

+/*
+** Implementation of fts5_insttoken() function.
+*/
+static void fts5InsttokenFunc(
+  sqlite3_context *pCtx,          /* Function call context */
+  int nArg,                       /* Number of args */
+  sqlite3_value **apArg           /* Function arguments */
+){
+  assert( nArg==1 );
+  sqlite3_result_value(pCtx, apArg[0]);
+  sqlite3_result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE);
+}
+
 /*
 ** Return true if zName is the extension on one of the shadow tables used
 ** by this module.
@@ -3781,10 +3800,17 @@ static int fts5Init(sqlite3 *db){
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_function(
          db, "fts5_locale", 2, 
-          SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE,
+          SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE|SQLITE_SUBTYPE,
          p, fts5LocaleFunc, 0, 0
      );
    }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_create_function(
+          db, "fts5_insttoken", 1, 
+          SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE,
+          p, fts5InsttokenFunc, 0, 0
+      );
+    }
  }

  /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file
@@ -1353,7 +1353,7 @@ static int fts5TriTokenize(
  char *zOut = aBuf;
  int ii;
  const unsigned char *zIn = (const unsigned char*)pText;
-  const unsigned char *zEof = &zIn[nText];
+  const unsigned char *zEof = (zIn ? &zIn[nText] : 0);
  u32 iCode = 0;
  int aStart[3];                  /* Input offset of each character in aBuf[] */

@@ -22,34 +22,40 @@ ifcapable !fts5 {
 }

 foreach_detail_mode $testprefix {
+foreach {tn insttoken} {
+  1 0
+  2 1
+} {
+reset_db

 sqlite3_fts5_register_origintext db
-do_execsql_test 1.0 {
+do_execsql_test $tn.1.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize="origintext unicode61", detail=%DETAIL%
  );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 }

-do_execsql_test 1.1 {
+do_execsql_test $tn.1.1 {
  INSERT INTO ft VALUES('Hello world');
 }

-do_execsql_test 1.2 {
+do_execsql_test $tn.1.2 {
  INSERT INTO ft(ft) VALUES('integrity-check');
 }

 proc b {x} { string map [list "\0" "."] $x }
 db func b b

-do_execsql_test 1.3 {
+do_execsql_test $tn.1.3 {
  select b(term) from vocab;
 } {
  hello.Hello
  world
 }

-do_execsql_test 1.4 {
+do_execsql_test $tn.1.4 {
  SELECT rowid FROM ft('Hello');
 } {1}

@@ -88,33 +94,34 @@ proc document {} {
 db func document document

 sqlite3_fts5_register_origintext db
-do_execsql_test 2.0 {
+do_execsql_test $tn.2.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize="origintext unicode61", detail=%DETAIL%
  );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 }

-do_test 2.1 {
+do_test $tn.2.1 {
  for {set ii 0} {$ii < 500} {incr ii} {
    execsql { INSERT INTO ft VALUES( document() ) }
  }
 } {}

-do_execsql_test 2.2 {
+do_execsql_test $tn.2.2 {
  INSERT INTO ft(ft) VALUES('integrity-check');
 }

-do_execsql_test 2.3 {
+do_execsql_test $tn.2.3 {
  INSERT INTO ft(ft, rank) VALUES('merge', 16);
 }

-do_execsql_test 2.4 {
+do_execsql_test $tn.2.4 {
  INSERT INTO ft(ft) VALUES('integrity-check');
 }

-do_execsql_test 2.5 {
+do_execsql_test $tn.2.5 {
  INSERT INTO ft(ft) VALUES('optimize');
 }

@@ -122,10 +129,11 @@ do_execsql_test 2.5 {
 reset_db

 sqlite3_fts5_register_origintext db
-do_execsql_test 3.0 {
+do_execsql_test $tn.3.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize="origintext unicode61", detail=%DETAIL%
  );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);

  INSERT INTO ft(rowid, x) VALUES(1, 'hello');
@@ -137,16 +145,17 @@ do_execsql_test 3.0 {
 #db func b b
 #execsql_pp { SELECT b(term) FROM vocab }

-do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1
-do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2
-do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
+do_execsql_test $tn.3.1.1 { SELECT rowid FROM ft('hello') } 1
+do_execsql_test $tn.3.1.2 { SELECT rowid FROM ft('Hello') } 2
+do_execsql_test $tn.3.1.3 { SELECT rowid FROM ft('HELLO') } 3

-do_execsql_test 3.2 {
+do_execsql_test $tn.3.2 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, 
      tokenize="origintext unicode61", 
      tokendata=1,
      detail=%DETAIL%
  );
+  INSERT INTO ft2(ft2, rank) VALUES('insttoken', $insttoken);
  CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);

  INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
@@ -160,11 +169,18 @@ do_execsql_test 3.2 {
 #db func b b
 #execsql_pp { SELECT b(term) FROM vocab }

-do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
-do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
-do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
+do_execsql_test $tn.3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
+do_execsql_test $tn.3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
+do_execsql_test $tn.3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}

-do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+do_execsql_test $tn.3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+
+do_execsql_test $tn.3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} {
+  3 2 1
+}
+do_execsql_test $tn.3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} {
+  3 2 1
+}

 #-------------------------------------------------------------------------
 #
@@ -176,36 +192,37 @@ proc querytoken {cmd iPhrase iToken} {
 }
 sqlite3_fts5_create_function db querytoken querytoken

-do_execsql_test 4.0 {
+do_execsql_test $tn.4.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
  );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  INSERT INTO ft VALUES('one two three four');
 }

-do_execsql_test 4.1 {
+do_execsql_test $tn.4.1 {
  SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
 } {1 two.TwO}
-do_execsql_test 4.2 {
+do_execsql_test $tn.4.2 {
  SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
 } {1 one}
-do_execsql_test 4.3 {
+do_execsql_test $tn.4.3 {
  SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
 } {1 two.TWO}

 if {"%DETAIL%"=="full"} {
  # Phrase queries are only supported for detail=full.
  #
-  do_execsql_test 4.4 {
+  do_execsql_test $tn.4.4 {
    SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
  } {1 three.ThreE}
-  do_catchsql_test 4.5 {
+  do_catchsql_test $tn.4.5 {
    SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
  } {1 SQLITE_RANGE}
-  do_catchsql_test 4.6 {
+  do_catchsql_test $tn.4.6 {
    SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
  } {1 SQLITE_RANGE}
-  do_catchsql_test 4.7 {
+  do_catchsql_test $tn.4.7 {
    SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
  } {1 SQLITE_RANGE}
 }
@@ -221,14 +238,15 @@ proc insttoken {cmd iIdx iToken} {
 sqlite3_fts5_create_function db insttoken insttoken
 fts5_aux_test_functions db

-do_execsql_test 5.0 {
+do_execsql_test $tn.5.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
  );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  INSERT INTO ft VALUES('one ONE One oNe oNE one');
 }

-do_execsql_test 5.1 {
+do_execsql_test $tn.5.1 {
  SELECT insttoken(ft, 0, 0), 
         insttoken(ft, 1, 0),
         insttoken(ft, 2, 0),
@@ -240,13 +258,37 @@ do_execsql_test 5.1 {
  one one.ONE one.One one.oNe one.oNE one
 }

-do_execsql_test 5.2 {
+do_execsql_test $tn.5.2 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft('on*');
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test $tn.5.3 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft(fts5_insttoken('on*'));
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test $tn.5.4 {
  SELECT insttoken(ft, 1, 0) FROM ft('one');
 } {
  one.ONE
 }

-do_execsql_test 5.3 {
+do_execsql_test $tn.5.5 {
  SELECT fts5_test_poslist(ft) FROM ft('one');
 } {
  {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
@@ -260,10 +302,11 @@ do_execsql_test 5.3 {
 #
 reset_db
 sqlite3_fts5_register_origintext db
-do_execsql_test 6.0 {
+do_execsql_test $tn.6.0 {
  CREATE VIRTUAL TABLE ft USING fts5( 
      x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0
  );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);

  INSERT INTO ft VALUES('One Two', 'Three two');
  INSERT INTO ft VALUES('three Three', 'one One');
@@ -279,34 +322,35 @@ proc tokens {cmd} {
 }
 sqlite3_fts5_create_function db tokens tokens

-do_execsql_test 6.1 {
+do_execsql_test $tn.6.1 {
  SELECT rowid, tokens(ft) FROM ft('One');
 } {1 one.One 2 one.One}

-do_execsql_test 6.2 {
+do_execsql_test $tn.6.2 {
  SELECT rowid, tokens(ft) FROM ft('on*');
-} {1 {{}} 2 {{} {}}}
+} {1 one.One 2 {one one.One}}

-do_execsql_test 6.3 {
+do_execsql_test $tn.6.3 {
  SELECT rowid, tokens(ft) FROM ft('Three*');
-} {1 {{}} 2 {{}}}
+} {1 three.Three 2 three.Three}

 fts5_aux_test_functions db
-do_catchsql_test 6.4 {
+do_catchsql_test $tn.6.4 {
  SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one');
 } {1 SQLITE_RANGE}

-do_catchsql_test 6.5 {
+do_catchsql_test $tn.6.5 {
  SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one');
 } {1 SQLITE_RANGE}

-do_catchsql_test 6.6 {
+do_catchsql_test $tn.6.6 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2);
 } {1 {malformed tokendata=... directive}}
-do_catchsql_test 6.7 {
+do_catchsql_test $tn.6.7 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11);
 } {1 {malformed tokendata=... directive}}

+}
 }

 finish_test
@@ -22,6 +22,11 @@ ifcapable !fts5 {
 }

 foreach_detail_mode $testprefix {
+  foreach {tn insttoken} {
+    1 0
+    2 1
+  } {
+
  reset_db

  sqlite3_fts5_register_origintext db
@@ -32,21 +37,25 @@ foreach_detail_mode $testprefix {
  }
  sqlite3_fts5_create_function db insttoken insttoken
  
-  do_execsql_test 1.0 {
+  do_execsql_test $tn.1.0 {
    CREATE VIRTUAL TABLE ft USING fts5(
        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
    );
  }
+
+  do_execsql_test $tn.1.0.1 {
+    INSERT INTO ft(ft, rank) VALUES('insttoken', 1);
+  }
  
-  do_execsql_test 1.1 {
+  do_execsql_test $tn.1.1 {
    INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
  }
  
-  do_execsql_test 1.2 {
+  do_execsql_test $tn.1.2 {
    SELECT fts5_test_poslist(ft) FROM ft('hello');
  } {{0.0.0 0.0.2 0.0.4}}

-  do_execsql_test 1.3 {
+  do_execsql_test $tn.1.3 {
    SELECT 
      insttoken(ft, 0, 0),
      insttoken(ft, 1, 0),
@@ -54,7 +63,15 @@ foreach_detail_mode $testprefix {
    FROM ft('hello');
  } {hello.Hello hello.HELLO hello}

-  do_execsql_test 1.4 {
+  do_execsql_test $tn.1.3.1 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hel*');
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test $tn.1.4 {
    SELECT 
      insttoken(ft, 0, 0),
      insttoken(ft, 1, 0),
@@ -62,7 +79,7 @@ foreach_detail_mode $testprefix {
    FROM ft('hello') ORDER BY rank;
  } {hello.Hello hello.HELLO hello}

-  do_execsql_test 1.5 {
+  do_execsql_test $tn.1.5 {
    CREATE VIRTUAL TABLE ft2 USING fts5(
        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
    );
@@ -71,11 +88,11 @@ foreach_detail_mode $testprefix {
    INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE');
  }

-  do_execsql_test 1.6 {
+  do_execsql_test $tn.1.6 {
    SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
  } {three.THREE 3 three 1 three 2}

-  do_execsql_test 1.7 {
+  do_execsql_test $tn.1.7 {
    INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
    INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
    INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
@@ -92,9 +109,32 @@ foreach_detail_mode $testprefix {
    INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
  }

-  do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
-  do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
+  do_execsql_test $tn.1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
+  do_execsql_test $tn.1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}

+  do_execsql_test $tn.2.0 {
+    CREATE VIRTUAL TABLE ft3 USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%,
+        prefix=2
+    );
+  }
+  do_execsql_test $tn.2.1 {
+    INSERT INTO ft3(rowid, x) VALUES(1, 'one');
+    INSERT INTO ft3(rowid, x) VALUES(2, 'ONE');
+    INSERT INTO ft3(rowid, x) VALUES(3, 'ONT');
+    INSERT INTO ft3(rowid, x) VALUES(4, 'on');
+    INSERT INTO ft3(rowid, x) VALUES(5, 'On');
+  }
+
+  do_execsql_test $tn.2.2 {
+    SELECT rowid FROM ft3('on*');
+  } {1 2 3 4 5}
+
+  do_execsql_test $tn.2.3 {
+    SELECT rowid, insttoken(ft3, 0, 0) FROM ft3('on*');
+  } {1 one 2 one.ONE 3 ont.ONT 4 on 5 on.On}
+
+  }
 }

 finish_test
@@ -0,0 +1,209 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext6
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc insert_data {tbl} {
+  db eval "
+  INSERT INTO $tbl (rowid, x, y) VALUES
+    (1, 'ChH BDd HhG efc BjJ BGi GBG FdD','ciJ AFf ADf fBJ fhC GFI JEH fcA'),
+    (2, 'deg AIG Fie jII cCd Hbf igF fEE','GeA Ija gJg EDc HFi DDI dCf aDd'),
+    (3, 'IJC hga deC Jfa Aeg hfh CcH dfb','ajD hgC Jaf IfH CHe jIG AjD adF'),
+    (4, 'FiH GJH IDA AiG bBc CGG Eih bIH','hHg JaH aii IHE Ggd gcH gji CGc'),
+    (5, 'ceg CAd jFI GAB BGg EeC IdH acG','bBC eIG ifH eDE Adj bjb GCj ebA'),
+    (6, 'Eac Fbh aFF Eea jeG EIj HCc JJH','hbd giE Gfe eiI dEF abE cJf cAb'),
+    (7, 'dic hAc jEC AiG FEF jHc HiD HBI','aEd ebE Gfi AJG EBA faj GiG jjE'),
+    (8, 'Fca iEe EgE jjJ gce ijf EGc EBi','gaI dhH bFg CFc HeC CjI Jfg ccH'),
+    (9, 'cfd iaa HCf iHJ HjG ffh ABb ibi','CfG bia Dai eii Ejg Jeg fCg hDb'),
+    (10, 'Jjf hJC IID HJj bGB EbJ cgg eBj','jci jhi JAF jIg Bei Bcd cAC AJd'),
+    (11, 'egG Cdi bFf fEB hfH jDH jia Efd','FAd eCg fAi aiC baC eJG acF iGE'),
+    (12, 'Ada Gde CJI ADG gJA Cbb ccF iAB','eAE ajC FBB ccd Jgh fJg ieg hGE'),
+    (13, 'gBb fDG Jdd HdD fiJ Bed Cig iGg','heC FeI iaj gdg ebB giC HaD FIe'),
+    (14, 'FiI iDd Ffe igI bgB EJf FHG hDF','cjC AeI abf Fah cbJ ffH jEb aib'),
+    (15, 'jaF hBI jIH Gdh FEc Fij hgj jFh','dGA ADH feh AAI AfJ DbC gBi hGH'),
+    (16, 'gjH BGg iGj aFE CAH edI idf HEH','hIf DDg fjB hGi cHF BCH FjG Bgd'),
+    (17, 'iaI JGH hji gcj Dda eeG jDd CBi','cHg jeh caG gIc feF ihG hgJ Abj'),
+    (18, 'jHI iDB eFf AiH EFB CDb IAj GbC','Ghe dEI gdI jai gib dAG BIa djb'),
+    (19, 'abI fHG Ccf aAc FDa fiC agF bdB','afi hde IgE bGF cfg DHD diE aca'),
+    (20, 'IFh eDJ jfh cDg dde JGJ GAf fIJ','IBa EfH faE aeI FIF baJ FGj EIH'),
+    (21, 'Dee bFC bBA dEI CEj aJI ghA dCH','hBA ddA HJh dfj egI Dij dFE bGE'),
+    (22, 'JFE BCj FgA afc Jda FGD iHJ HDh','eAI jHe BHD Gah bbD Bgj gbh eGB'),
+    (23, 'edE CJE FjG aFI edA Cea FId iFe','ABG jcA ddj EEc Dcg hAI agA biA'),
+    (24, 'AgE cfc eef cGh aFB DcH efJ hcH','eGF HaB diG fgi bdc iGJ FGJ fFB'),
+    (25, 'aCa AgI GhC DDI hGJ Hgc Gcg bbG','iID Fga jHa jIj idj DFD bAC AFJ'),
+    (26, 'gjC JGh Fge faa eCA iGG gHE Gai','bDi hFE BbI DHD Adb Fgi hCa Hij'),
+    (27, 'Eji jEI jhF DFC afH cDh AGc dHA','IDe GcA ChF DIb Bif HfH agD DGh'),
+    (28, 'gDD AEE Dfg ICf Cbi JdE jgH eEi','eEb dBG FDE jgf cAI FaJ jaA cDd'),
+    (29, 'cbe Gec hgB Egi bca dHg bAJ jBf','EFB DgD GJc fDb EeE bBA GFC Hbe'),
+    (30, 'Adc eHB afI hDc Bhh baE hcJ BBd','JAH deg bcF Dab Bgj Gbb JHi FIB'),
+    (31, 'agF dIj AJJ Hfg cCG hED Igc fHC','JEf eia dHf Ggc Agj geD bEE Gei'),
+    (32, 'DAd cCe cbJ FjG gJe gba dJA GCf','eAf hFc bGE ABI hHA IcE abF CCE'),
+    (33, 'fFh jJe DhJ cDJ EBi AfD eFI IhG','fEG GCc Bjd EFF ggg CFe EHd ciB'),
+    (34, 'Ejb BjI eAF HaD eEJ FaG Eda AHC','Iah hgD EJG fdD cIE Daj IFf eJh'),
+    (35, 'aHG eCe FjA djJ dAJ jiJ IaE GGB','Acg iEF JfB FIC Eei ggj dic Iii'),
+    (36, 'Fdb EDF GaF JjB ehH IgC hgi DCG','cag DHI Fah hAJ bbh egG Hia hgJ'),
+    (37, 'HGg icC JEC AFJ Ddh dhi hfC Ich','fEg bED Bff hCJ EiA cIf bfG cGA'),
+    (38, 'aEJ jGI BCi FaA ebA BHj cIJ GcC','dCH ADd bGB cFE AgF geD cbG jIc'),
+    (39, 'JFB bBi heA BFA hgB Ahj EIE CgI','EIJ JFG FJE GeA Hdg HeH ACh GiA'),
+    (40, 'agB DDC CED igC Dfc DhI eiC fHi','dAB dcg iJF cej Fcc cAc AfB Fdd'),
+    (41, 'BdF DHj Ege hcG DEd eFa dCf gBb','FBG ChB cej iGd Hbh fCc Ibe Abh'),
+    (42, 'Bgc DjI cbC jGD bdb hHB IJA IJH','heg cii abb IGf eDe hJc dii fcE'),
+    (43, 'fhf ECa FiA aDh Jbf CiB Jhe ajD','GFE bIF aeD gDE BIE Jea DfC BEc'),
+    (44, 'GjE dBj DbJ ICF aDh EEH Ejb jFb','dJj aEc IBg bEG Faf fjA hjf FAF'),
+    (45, 'BfA efd IIJ AHG dDF eGg dIJ Gcb','Bfj jeb Ahc dAE ACH Dfb ieb dhC'),
+    (46, 'Ibj ege geC dJh CIi hbD EAG fGA','DEb BFe Bjg FId Fhg HeF JAc BbE'),
+    (47, 'dhB afC hgG bEJ aIe Cbe iEE JCD','bdg Ajc FGA jbh Jge iAj fIA jbE'),
+    (48, 'egH iDi bfH iiI hGC jFF Hfd AHB','bjE Beb iCc haB gIH Dea bga dfd'),
+    (49, 'jgf chc jGc Baj HBb jdE hgh heI','FFB aBd iEB EIG HGf Bbj EIi JbI'),
+    (50, 'jhe EGi ajA fbH geh EHe FdC bij','jDE bBC gbH HeE dcH iBH IFE AHi'),
+    (51, 'aCb JiD cgJ Bjj iAI Hbe IAF FhH','ijf bhE Jdf FED dCH bbG HcJ ebH');
+  "
+}
+
+foreach_detail_mode $testprefix {
+foreach external {0 1 2} {
+  reset_db
+
+  proc tokens {cmd} { 
+    set ret [list]
+    for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+      set txt [$cmd xInstToken $iTok 0]
+      set txt [string map [list "\0" "."] $txt]
+      lappend ret $txt
+    }
+    set ret
+  }
+  sqlite3_fts5_create_function db tokens tokens
+  sqlite3_fts5_register_origintext db
+
+  set E(0) internal
+  set E(1) external
+  set E(2) contentless
+  set e $E($external)
+
+  db eval { CREATE TABLE ex(x, y) }
+  switch -- $external {
+    0 {
+      do_execsql_test 1.$e.0 {
+        CREATE VIRTUAL TABLE ft USING fts5(
+            x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+        );
+      }
+    }
+
+    1 {
+      do_execsql_test 1.$e.0 {
+        CREATE VIRTUAL TABLE ft USING fts5(
+            x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%,
+            content=ex
+        );
+      }
+    }
+
+    2 {
+      do_execsql_test 1.$e.0 {
+        CREATE VIRTUAL TABLE ft USING fts5(
+            x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%,
+            content=
+        );
+      }
+    }
+  }
+  insert_data ex
+  insert_data ft
+  
+  proc prefixquery {prefix bInst bYOnly} {
+    set ret [list]
+    db eval { SELECT rowid, x, y FROM ex ORDER BY rowid } {
+      set row [list]
+      set bSeen 0
+
+      set T [concat $x $y]
+      if {$bYOnly} { set T $y }
+
+      foreach w $T {
+        if {[string match -nocase $prefix $w]} {
+          set bSeen 1
+          if {$bInst} {
+            set v [string tolower $w]
+            if {$w != $v} { append v ".$w" }
+            lappend row $v
+          }
+        }
+      }
+  
+      if {$bSeen} {
+        lappend ret $rowid
+        lappend ret $row
+      }
+    }
+  
+    set ret
+  }
+  
+  proc do_prefixquery_test {tn prefix} {
+    set bInst [expr {$::e!="contentless" || "%DETAIL%"=="full"}]
+    set expect [prefixquery $prefix $bInst 0]
+    set expect2 [prefixquery $prefix $bInst 1]
+
+    uplevel [list do_execsql_test $tn.1 "
+        SELECT rowid, tokens(ft) FROM ft('$prefix')
+    " $expect]
+    uplevel [list do_execsql_test $tn.2 "
+        SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('$prefix'))
+    " $expect]
+    db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) }
+    uplevel [list do_execsql_test $tn.3 "
+        SELECT rowid, tokens(ft) FROM ft('$prefix')
+    " $expect]
+    db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) }
+
+    if {"%DETAIL%"!="none"} {
+      uplevel [list do_execsql_test $tn.4 "
+          SELECT rowid, tokens(ft) FROM ft('y: $prefix')
+      " $expect2]
+      uplevel [list do_execsql_test $tn.5 "
+          SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('y: $prefix'))
+      " $expect2]
+      db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) }
+      uplevel [list do_execsql_test $tn.6 "
+          SELECT rowid, tokens(ft) FROM ft('y: $prefix')
+      " $expect2]
+      db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) }
+    }
+  }
+  
+  do_prefixquery_test 1.$e.1 a*
+  do_prefixquery_test 1.$e.2 b*
+  do_prefixquery_test 1.$e.3 c*
+  do_prefixquery_test 1.$e.4 d*
+  do_prefixquery_test 1.$e.5 e*
+  do_prefixquery_test 1.$e.6 f*
+  do_prefixquery_test 1.$e.7 g*
+  do_prefixquery_test 1.$e.8 h*
+  do_prefixquery_test 1.$e.9 i*
+  do_prefixquery_test 1.$e.10 j*
+}}
+
+
+
+finish_test
+
@@ -0,0 +1,105 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokendata
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+  sqlite3_fts5_register_origintext db
+  fts5_aux_test_functions db
+  proc b {x} { string map [list "\0" "."] $x }
+  db func b b
+
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(a, b, tokendata=1,
+      tokenize="origintext unicode61", detail=%DETAIL%
+    );
+    CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+  }
+
+  do_execsql_test 1.1 {
+    INSERT INTO ft(rowid, a, b) VALUES
+      (1, 'Pedagog Pedal Pedant', 'Peculier Day Today'),
+      (2, 'Pedant pedantic pecked', 'Peck Penalize Pen');
+
+    INSERT INTO ft(rowid, a, b) VALUES
+      (3, 'Penalty Pence Penciled', 'One Two Three'),
+      (4, 'Pedant Pedal Pedant', 'Peculier Day Today');
+  }
+
+  do_execsql_test 1.2 {
+    SELECT DISTINCT b(term) FROM vocab
+  } {
+    day.Day one.One peck.Peck pecked peculier.Peculier pedagog.Pedagog
+    pedal.Pedal pedant.Pedant pedantic pen.Pen penalize.Penalize 
+    penalty.Penalty pence.Pence penciled.Penciled three.Three 
+    today.Today two.Two
+  }
+
+  do_execsql_test 1.3.1 {
+    SELECT rowid FROM ft('pe*')
+  } {
+    1 2 3 4
+  }
+
+  do_execsql_test 1.3.2 {
+    SELECT rowid FROM ft('pe*') ORDER BY rowid DESC
+  } {
+    4 3 2 1
+  }
+  
+  if {"%DETAIL%"!="none"} {
+    do_execsql_test 1.3.3 {
+      SELECT rowid FROM ft WHERE a MATCH 'pe*' ORDER BY rowid DESC
+    } {
+      4 3 2 1
+    }
+  }
+
+  do_execsql_test 1.4 {
+    SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pedant')
+  } {
+    1 pedant.Pedant
+    2 pedant.Pedant
+    4 pedant.Pedant
+  }
+
+  do_execsql_test 1.5 {
+    SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pe*')
+  } {
+    1 pedagog.Pedagog
+    2 pedant.Pedant
+    3 penalty.Penalty
+    4 pedant.Pedant
+  }
+
+  do_execsql_test 1.6 {
+    SELECT rowid, fts5_test_poslist(ft) FROM ft('pe*')
+  } {
+    1 {0.0.0 0.0.1 0.0.2 0.1.0} 
+    2 {0.0.0 0.0.1 0.0.2 0.1.0 0.1.1 0.1.2}
+    3 {0.0.0 0.0.1 0.0.2}
+    4 {0.0.0 0.0.1 0.0.2 0.1.0}
+  }
+}
+
+finish_test
+
@@ -1248,8 +1248,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
      let st = this.db.prepare(
        new TextEncoder('utf-8').encode("select 3 as a")
      );
-      //debug("statement =",st);
-      T.assert( !this.progressHandlerCount );
      let rc;
      try {
        T.assert(wasm.isPtr(st.pointer))
@@ -18,7 +18,6 @@
 # invoked. This file will use defaults, very possibly invalid, for any
 # which are not defined.
 ########################################################################
-.POSIX: #maintenance reminder: X:=Y is not POSIX-portable
 all:
 #
 # $(TOP) =
@@ -106,6 +105,9 @@ TCLSH_CMD ?= tclsh
 # JIMSH requires a leading path component, even if it's ./, so that it
 # can be used as a shell command.
 #
+# On Windows platforms, if -DHAVE_REALPATH does not work then try
+# -DHAVE__FULLPATH (note the double-underscore).
+#
 CFLAGS.jimsh ?= -DHAVE_REALPATH
 JIMSH ?= ./jimsh$(T.exe)
 #
@@ -120,12 +122,12 @@ B.tclsh ?= $(JIMSH)

 #
 # Autotools-conventional vars which are (in this tree) used only by
-# package installation rules.
+# package installation rules and for generating sqlite3.pc (pkg-config
+# data file).
 #
 # The following ${XYZdir} vars are provided for the sake of clients
 # who expect to be able to override these using autotools-conventional
-# dir name vars. In this build they apply only to installation-related
-# rules.
+# dir name vars.
 #
 prefix      ?= /usr/local
 datadir     ?= $(prefix)/share
@@ -160,10 +162,10 @@ LDFLAGS.math ?= -lm
 LDFLAGS.rpath ?= -Wl,-rpath -Wl,$(prefix)/lib
 LDFLAGS.pthread ?= -lpthread
 LDFLAGS.dlopen ?= -ldl
-LDFLAGS.shobj ?= -shared
+LDFLAGS.shlib ?= -shared
 LDFLAGS.icu ?= # -licui18n -licuuc -licudata
 CFLAGS.icu ?=
-LDFLAGS.soname.libsqlite3 ?=
+LDFLAGS.libsqlite3.soname ?= # see https://sqlite.org/src/forumpost/5a3b44f510df8ded
 # libreadline (or a workalike):
 # To activate readline in the shell: SHELL_OPT = -DHAVE_READLINE=1
 LDFLAGS.readline ?= -lreadline # these vary across platforms
@@ -203,9 +205,25 @@ ENABLE_STATIC ?= 1
 #
 # 1 if the amalgamation (sqlite3.c/h) should be built/used, otherwise
 # the library is built from all of its original source files.
+# Certaint tools, like sqlite3$(T.exe), require the amalgamation and
+# will ignore this preference.
 #
 USE_AMALGAMATION ?= 1
 #
+# $(LINK_TOOLS_DYNAMICALLY)
+#
+# If 1, certain binaries which typically statically link against
+# libsqlite3 or its component object files will instead link against
+# the DLL. The caveat is that running such builds from the source tree
+# may require that the user specifically prepend "." to their
+# $LD_LIBRARY_PATH so that the dynamic linker does not pick up a
+# libsqlite3.so from outside the source tree. Alternately, symlinking
+# the in-build-tree $(libsqlite3.SO) to some dir in the system's
+# library path will work for giving the apps access to the in-tree
+# DLL.
+#
+LINK_TOOLS_DYNAMICALLY ?= 0
+#
 # $(AMALGAMATION_GEN_FLAGS) =
 #
 # Optional flags for the amalgamation generator.
@@ -217,16 +235,21 @@ AMALGAMATION_GEN_FLAGS ?= --linemacros=0
 # Preprocessor flags for enabling and disabling specific libsqlite3
 # features (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). The same set of OMIT
 # and ENABLE flags must be passed to the LEMON parser generator and
-# the mkkeywordhash tool as well.
+# the mkkeywordhash tool as well. This is normally set by the
+# configure process, and passing a custom value to a
+# coonfigure-filtered Makefile may not work.
 #
-# Add OPTIONS=... on the make command line to append additional options
-# to the OPT_FEATURE_FLAGS. Note that some flags only work if the
-# build is specifically configured to account for them. Adding them
-# later, when compiling the amalgamation, may or may not work.
+# When using the canonical makefile, add $(OPTIONS)=... on the make
+# command line to append additional options to the
+# $(OPT_FEATURE_FLAGS). Note that some flags, because they influence
+# generation of the SQL parser, only work if the build is specifically
+# configured to account for them. Adding them later, when compiling
+# the amalgamation separately, may or may not work.
 #
-# TO CLARIFY: OPTS=... has historically been expected in some
-# contexts, and is distinctly different from OPTIONS and
-# OPT_FEATURE_FLAGS, but its name is confusingly close to $(OPTIONS).
+# $(OPTS)=... is another way of influencing C compilation. It is
+# distinctly separate from $(OPTIONS) and $(OPT_FEATURE_FLAGS) but,
+# like those, $(OPTS) applies to all invocations of $(T.cc). The
+# configure process does not set either of $(OPTIONS) or $(OPTS).
 #
 OPT_FEATURE_FLAGS ?=
 #
@@ -238,15 +261,17 @@ SHELL_OPT ?=
 #
 # TCL_CONFIG_SH must, for some of the build targets, refer to a valid
 # tclConfig.sh. That script will be used to populate most of the other
-# TCL-related vars the build needs.
+# TCL-related vars the build needs. The core library does not require
+# TCL, but TCL is needed for running tests and certain tools, e.g.
+# sqlite3_analyzer.
 #
 TCL_CONFIG_SH ?=
 #
 # $(HAVE_WASI_SDK) =
 #
-# 1 when building with the WASI SDK. This disables certain build
-# targets. It is expected that the invoker assigns CC to the wasi-sdk
-# CC.
+# Set to 1 when building with the WASI SDK. This disables certain
+# build targets. It is expected that the invoker sets $(CC), $(LD),
+# and $(AR) to their counterparts from the wasi-sdk.
 #
 HAVE_WASI_SDK ?= 0
 #
@@ -302,22 +327,21 @@ T.cc += $(CFLAGS.core) $(CFLAGS.env)
 # The legacy build applied such LDFLAGS to all link operations for all
 # deliverables. The 3.48+ build applies them (as of this writing) more
 # selectively: search this file LDFLAGS.configure to see where they're
-# set. As of this writing, they only affect targets which use
-# $(LDFLAGS.libsqlite3) - see that var's docs for details.
+# set.
 #
 LDFLAGS.configure ?=

 #
 # The difference between $(OPT_FEATURE_FLAGS) and $(OPTS) is that the
-# former is historically provided by the configure script, whereas the
-# latter is intended to be provided as arguments to the make
+# former is historically provided by the configure script, whereas
+# $(OPTS) is intended to be provided as arguments to the make
 # invocation.
 #
 T.cc += $(OPT_FEATURE_FLAGS)

 #
 # Add in any optional global compilation flags on the make command
-# line ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
+# line i.e.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
 #
 T.cc += $(OPTS)

@@ -334,12 +358,6 @@ INSTALL.noexec = $(INSTALL) -m 0644
 #
 T.compile = $(T.cc) $(T.compile.extras)

-#
-# $(CFLAGS.libsqlite3) must contain any CFLAGS which are relevant for
-# compiling the library's own sources, including (sometimes) when
-# compiling sqlite3.c directly in to another app.
-#
-CFLAGS.libsqlite3 ?=
 #
 # $(T.cc.sqlite) is $(T.cc) plus any flags which are desired for the
 # library as a whole, but not necessarily needed for every binary. It
@@ -375,7 +393,7 @@ T.link = $(T.cc.sqlite) $(T.link.extras)
 #
 # $(T.link.shared) = $(T.link) invocation specifically for shared libraries
 #
-T.link.shared = $(T.link) $(LDFLAGS.shobj)
+T.link.shared = $(T.link) $(LDFLAGS.shlib)

 #
 # $(LDFLAGS.libsqlite3) should be used with any deliverable for which
@@ -894,6 +912,9 @@ TESTOPTS = --verbose=file --output=test-out.txt
 #
 # Extra compiler options for various shell tools
 #
+# Note that some of these will only apply when embedding sqlite3.c
+# into the shell, as these flags are not otherwise passed on to the
+# library.
 SHELL_OPT += -DSQLITE_DQS=0
 SHELL_OPT += -DSQLITE_ENABLE_FTS4
 #SHELL_OPT += -DSQLITE_ENABLE_FTS5
@@ -1018,7 +1039,7 @@ T.link.tcl = $(T.tcl.env.source); $(T.link)
 # all that automatic generation.
 #
 .target_source: $(MAKE_SANITY_CHECK) $(SRC) $(TOP)/tool/vdbe-compress.tcl \
-    fts5.c $(B.tclsh) # has_tclsh84
+    fts5.c $(B.tclsh)
 	rm -rf tsrc
 	mkdir tsrc
 	cp -f $(SRC) tsrc
@@ -1044,19 +1065,19 @@ mksourceid$(B.exe): $(MAKE_SANITY_CHECK) $(TOP)/tool/mksourceid.c

 sqlite3.h: $(MAKE_SANITY_CHECK) $(TOP)/src/sqlite.h.in \
    $(TOP)/manifest mksourceid$(B.exe) \
-		$(TOP)/VERSION $(B.tclsh) # has_tclsh84
+		$(TOP)/VERSION $(B.tclsh)
 	$(B.tclsh) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h

 sqlite3.c:	.target_source sqlite3.h $(TOP)/tool/mksqlite3c.tcl src-verify$(B.exe) \
-		$(B.tclsh) # has_tclsh84
+		$(B.tclsh)
 	$(B.tclsh) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_GEN_FLAGS) $(EXTRA_SRC)
 	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .

-sqlite3r.h: sqlite3.h $(B.tclsh) # has_tclsh84
+sqlite3r.h: sqlite3.h $(B.tclsh)
 	$(B.tclsh) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h

-sqlite3r.c: sqlite3.c sqlite3r.h $(B.tclsh) # has_tclsh84
+sqlite3r.c: sqlite3.c sqlite3r.h $(B.tclsh)
 	cp $(TOP)/ext/recover/sqlite3recover.c tsrc/
 	cp $(TOP)/ext/recover/sqlite3recover.h tsrc/
 	cp $(TOP)/ext/recover/dbdata.c tsrc/
@@ -1073,255 +1094,255 @@ sqlite3ext.h: .target_source
 #
 DEPS_OBJ_COMMON = $(MAKE_SANITY_CHECK) $(HDR)
 parse.o:	parse.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c parse.c
+	$(T.cc.sqlite) -c parse.c

 opcodes.o:	opcodes.c
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c opcodes.c
+	$(T.cc.sqlite) -c opcodes.c

 # Rules to build individual *.o files from files in the src directory.
 #
 alter.o:	$(TOP)/src/alter.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/alter.c
+	$(T.cc.sqlite) -c $(TOP)/src/alter.c

 analyze.o:	$(TOP)/src/analyze.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/analyze.c
+	$(T.cc.sqlite) -c $(TOP)/src/analyze.c

 attach.o:	$(TOP)/src/attach.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/attach.c
+	$(T.cc.sqlite) -c $(TOP)/src/attach.c

 auth.o:	$(TOP)/src/auth.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/auth.c
+	$(T.cc.sqlite) -c $(TOP)/src/auth.c

 backup.o:	$(TOP)/src/backup.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/backup.c
+	$(T.cc.sqlite) -c $(TOP)/src/backup.c

 bitvec.o:	$(TOP)/src/bitvec.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/bitvec.c
+	$(T.cc.sqlite) -c $(TOP)/src/bitvec.c

 btmutex.o:	$(TOP)/src/btmutex.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/btmutex.c
+	$(T.cc.sqlite) -c $(TOP)/src/btmutex.c

 btree.o:	$(TOP)/src/btree.c $(DEPS_OBJ_COMMON) $(TOP)/src/pager.h
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/btree.c
+	$(T.cc.sqlite) -c $(TOP)/src/btree.c

 build.o:	$(TOP)/src/build.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/build.c
+	$(T.cc.sqlite) -c $(TOP)/src/build.c

 callback.o:	$(TOP)/src/callback.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/callback.c
+	$(T.cc.sqlite) -c $(TOP)/src/callback.c

 complete.o:	$(TOP)/src/complete.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/complete.c
+	$(T.cc.sqlite) -c $(TOP)/src/complete.c

 ctime.o:	$(TOP)/src/ctime.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/ctime.c
+	$(T.cc.sqlite) -c $(TOP)/src/ctime.c

 date.o:	$(TOP)/src/date.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/date.c
+	$(T.cc.sqlite) -c $(TOP)/src/date.c

 dbpage.o:	$(TOP)/src/dbpage.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/dbpage.c
+	$(T.cc.sqlite) -c $(TOP)/src/dbpage.c

 dbstat.o:	$(TOP)/src/dbstat.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/dbstat.c
+	$(T.cc.sqlite) -c $(TOP)/src/dbstat.c

 delete.o:	$(TOP)/src/delete.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/delete.c
+	$(T.cc.sqlite) -c $(TOP)/src/delete.c

 expr.o:	$(TOP)/src/expr.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/expr.c
+	$(T.cc.sqlite) -c $(TOP)/src/expr.c

 fault.o:	$(TOP)/src/fault.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/fault.c
+	$(T.cc.sqlite) -c $(TOP)/src/fault.c

 fkey.o:	$(TOP)/src/fkey.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/fkey.c
+	$(T.cc.sqlite) -c $(TOP)/src/fkey.c

 func.o:	$(TOP)/src/func.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/func.c
+	$(T.cc.sqlite) -c $(TOP)/src/func.c

 global.o:	$(TOP)/src/global.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/global.c
+	$(T.cc.sqlite) -c $(TOP)/src/global.c

 hash.o:	$(TOP)/src/hash.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/hash.c
+	$(T.cc.sqlite) -c $(TOP)/src/hash.c

 insert.o:	$(TOP)/src/insert.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/insert.c
+	$(T.cc.sqlite) -c $(TOP)/src/insert.c

 json.o:	$(TOP)/src/json.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/json.c
+	$(T.cc.sqlite) -c $(TOP)/src/json.c

 legacy.o:	$(TOP)/src/legacy.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/legacy.c
+	$(T.cc.sqlite) -c $(TOP)/src/legacy.c

 loadext.o:	$(TOP)/src/loadext.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/loadext.c
+	$(T.cc.sqlite) -c $(TOP)/src/loadext.c

 main.o:	$(TOP)/src/main.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/main.c
+	$(T.cc.sqlite) -c $(TOP)/src/main.c

 malloc.o:	$(TOP)/src/malloc.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/malloc.c
+	$(T.cc.sqlite) -c $(TOP)/src/malloc.c

 mem0.o:	$(TOP)/src/mem0.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mem0.c
+	$(T.cc.sqlite) -c $(TOP)/src/mem0.c

 mem1.o:	$(TOP)/src/mem1.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mem1.c
+	$(T.cc.sqlite) -c $(TOP)/src/mem1.c

 mem2.o:	$(TOP)/src/mem2.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mem2.c
+	$(T.cc.sqlite) -c $(TOP)/src/mem2.c

 mem3.o:	$(TOP)/src/mem3.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mem3.c
+	$(T.cc.sqlite) -c $(TOP)/src/mem3.c

 mem5.o:	$(TOP)/src/mem5.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mem5.c
+	$(T.cc.sqlite) -c $(TOP)/src/mem5.c

 memdb.o:	$(TOP)/src/memdb.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/memdb.c
+	$(T.cc.sqlite) -c $(TOP)/src/memdb.c

 memjournal.o:	$(TOP)/src/memjournal.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/memjournal.c
+	$(T.cc.sqlite) -c $(TOP)/src/memjournal.c

 mutex.o:	$(TOP)/src/mutex.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mutex.c
+	$(T.cc.sqlite) -c $(TOP)/src/mutex.c

 mutex_noop.o:	$(TOP)/src/mutex_noop.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mutex_noop.c
+	$(T.cc.sqlite) -c $(TOP)/src/mutex_noop.c

 mutex_unix.o:	$(TOP)/src/mutex_unix.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mutex_unix.c
+	$(T.cc.sqlite) -c $(TOP)/src/mutex_unix.c

 mutex_w32.o:	$(TOP)/src/mutex_w32.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/mutex_w32.c
+	$(T.cc.sqlite) -c $(TOP)/src/mutex_w32.c

 notify.o:	$(TOP)/src/notify.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/notify.c
+	$(T.cc.sqlite) -c $(TOP)/src/notify.c

 pager.o:	$(TOP)/src/pager.c $(DEPS_OBJ_COMMON) $(TOP)/src/pager.h
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/pager.c
+	$(T.cc.sqlite) -c $(TOP)/src/pager.c

 pcache.o:	$(TOP)/src/pcache.c $(DEPS_OBJ_COMMON) $(TOP)/src/pcache.h
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/pcache.c
+	$(T.cc.sqlite) -c $(TOP)/src/pcache.c

 pcache1.o:	$(TOP)/src/pcache1.c $(DEPS_OBJ_COMMON) $(TOP)/src/pcache.h
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/pcache1.c
+	$(T.cc.sqlite) -c $(TOP)/src/pcache1.c

 os.o:	$(TOP)/src/os.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/os.c
+	$(T.cc.sqlite) -c $(TOP)/src/os.c

 os_kv.o:	$(TOP)/src/os_kv.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/os_kv.c
+	$(T.cc.sqlite) -c $(TOP)/src/os_kv.c

 os_unix.o:	$(TOP)/src/os_unix.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/os_unix.c
+	$(T.cc.sqlite) -c $(TOP)/src/os_unix.c

 os_win.o:	$(TOP)/src/os_win.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/os_win.c
+	$(T.cc.sqlite) -c $(TOP)/src/os_win.c

 pragma.o:	$(TOP)/src/pragma.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/pragma.c
+	$(T.cc.sqlite) -c $(TOP)/src/pragma.c

 prepare.o:	$(TOP)/src/prepare.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/prepare.c
+	$(T.cc.sqlite) -c $(TOP)/src/prepare.c

 printf.o:	$(TOP)/src/printf.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/printf.c
+	$(T.cc.sqlite) -c $(TOP)/src/printf.c

 random.o:	$(TOP)/src/random.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/random.c
+	$(T.cc.sqlite) -c $(TOP)/src/random.c

 resolve.o:	$(TOP)/src/resolve.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/resolve.c
+	$(T.cc.sqlite) -c $(TOP)/src/resolve.c

 rowset.o:	$(TOP)/src/rowset.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/rowset.c
+	$(T.cc.sqlite) -c $(TOP)/src/rowset.c

 select.o:	$(TOP)/src/select.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/select.c
+	$(T.cc.sqlite) -c $(TOP)/src/select.c

 status.o:	$(TOP)/src/status.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/status.c
+	$(T.cc.sqlite) -c $(TOP)/src/status.c

 sqlite3.o:	sqlite3.h sqlite3.c
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c sqlite3.c
+	$(T.cc.sqlite) -c sqlite3.c

 table.o:	$(TOP)/src/table.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/table.c
+	$(T.cc.sqlite) -c $(TOP)/src/table.c

 threads.o:	$(TOP)/src/threads.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/threads.c
+	$(T.cc.sqlite) -c $(TOP)/src/threads.c

 tokenize.o:	$(TOP)/src/tokenize.c keywordhash.h $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/tokenize.c
+	$(T.cc.sqlite) -c $(TOP)/src/tokenize.c

 treeview.o:	$(TOP)/src/treeview.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/treeview.c
+	$(T.cc.sqlite) -c $(TOP)/src/treeview.c

 trigger.o:	$(TOP)/src/trigger.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/trigger.c
+	$(T.cc.sqlite) -c $(TOP)/src/trigger.c

 update.o:	$(TOP)/src/update.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/update.c
+	$(T.cc.sqlite) -c $(TOP)/src/update.c

 upsert.o:	$(TOP)/src/upsert.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/upsert.c
+	$(T.cc.sqlite) -c $(TOP)/src/upsert.c

 utf.o:	$(TOP)/src/utf.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/utf.c
+	$(T.cc.sqlite) -c $(TOP)/src/utf.c

 util.o:	$(TOP)/src/util.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/util.c
+	$(T.cc.sqlite) -c $(TOP)/src/util.c

 vacuum.o:	$(TOP)/src/vacuum.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vacuum.c
+	$(T.cc.sqlite) -c $(TOP)/src/vacuum.c

 vdbe.o:	$(TOP)/src/vdbe.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbe.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbe.c

 vdbeapi.o:	$(TOP)/src/vdbeapi.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbeapi.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbeapi.c

 vdbeaux.o:	$(TOP)/src/vdbeaux.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbeaux.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbeaux.c

 vdbeblob.o:	$(TOP)/src/vdbeblob.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbeblob.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbeblob.c

 vdbemem.o:	$(TOP)/src/vdbemem.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbemem.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbemem.c

 vdbesort.o:	$(TOP)/src/vdbesort.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbesort.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbesort.c

 vdbetrace.o:	$(TOP)/src/vdbetrace.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbetrace.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbetrace.c

 vdbevtab.o:	$(TOP)/src/vdbevtab.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vdbevtab.c
+	$(T.cc.sqlite) -c $(TOP)/src/vdbevtab.c

 vtab.o:	$(TOP)/src/vtab.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/vtab.c
+	$(T.cc.sqlite) -c $(TOP)/src/vtab.c

 wal.o:	$(TOP)/src/wal.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/wal.c
+	$(T.cc.sqlite) -c $(TOP)/src/wal.c

 walker.o:	$(TOP)/src/walker.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/walker.c
+	$(T.cc.sqlite) -c $(TOP)/src/walker.c

 where.o:	$(TOP)/src/where.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/where.c
+	$(T.cc.sqlite) -c $(TOP)/src/where.c

 wherecode.o:	$(TOP)/src/wherecode.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/wherecode.c
+	$(T.cc.sqlite) -c $(TOP)/src/wherecode.c

 whereexpr.o:	$(TOP)/src/whereexpr.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/whereexpr.c
+	$(T.cc.sqlite) -c $(TOP)/src/whereexpr.c

 window.o:	$(TOP)/src/window.c $(DEPS_OBJ_COMMON)
-	$(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/window.c
+	$(T.cc.sqlite) -c $(TOP)/src/window.c

 tclsqlite.o:	$(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON)
 	$(T.compile.tcl) -DUSE_TCL_STUBS=1 $$TCL_INCLUDE_SPEC \
@@ -1343,11 +1364,11 @@ tcl: tclsqlite3$(T.exe)-$(HAVE_TCL)

 # Rules to build opcodes.c and opcodes.h
 #
-opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl $(B.tclsh) # has_tclsh84
+opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl $(B.tclsh)
 	$(B.tclsh) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c

 opcodes.h:	parse.h $(TOP)/src/vdbe.c \
-		$(TOP)/tool/mkopcodeh.tcl $(B.tclsh) # has_tclsh84
+		$(TOP)/tool/mkopcodeh.tcl $(B.tclsh)
 	cat parse.h $(TOP)/src/vdbe.c | $(B.tclsh) $(TOP)/tool/mkopcodeh.tcl >opcodes.h

 # Rules to build parse.c and parse.h - the outputs of lemon.
@@ -1358,7 +1379,7 @@ parse.c:	$(TOP)/src/parse.y lemon$(B.exe)
 	cp $(TOP)/src/parse.y .
 	./lemon$(B.exe) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y

-sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION $(B.tclsh) # has_tclsh84
+sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION $(B.tclsh)
 	echo '#ifndef SQLITE_RESOURCE_VERSION' >$@
 	echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@
 	cat $(TOP)/VERSION | $(B.tclsh) $(TOP)/tool/replace.tcl exact . , >>$@
@@ -1372,7 +1393,7 @@ keywordhash.h:	mkkeywordhash$(B.exe)
 #
 # sqlite3.c split into many smaller files.
 #
-sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl $(B.tclsh) # has_tclsh84
+sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl $(B.tclsh)
 	$(B.tclsh) $(TOP)/tool/split-sqlite3c.tcl

 #
@@ -1389,8 +1410,8 @@ all: lib
 # Dynamic libsqlite3
 #
 $(libsqlite3.SO):	$(LIBOBJ)
-	$(T.link.shared) -o $@ $(LIBOBJ) $(LDFLAGS.soname.libsqlite3) \
-		$(LDFLAGS.libsqlite3) $(LDFLAGS.libsqlite3.soname)
+	$(T.link.shared) -o $@ $(LIBOBJ) $(LDFLAGS.libsqlite3) \
+		$(LDFLAGS.libsqlite3.soname)
 $(libsqlite3.SO)-1: $(libsqlite3.SO)
 $(libsqlite3.SO)-0 $(libsqlite3.SO)-:
 so: $(libsqlite3.SO)-$(ENABLE_SHARED)
@@ -1401,9 +1422,12 @@ all: so
 # and create symlinks which point to it:
 #
 # - libsqlite3.so.$(PACKAGE_VERSION)
-# - libsqlite3.so.3 =symlink-> libsqlite3.so.$(PACKAGE_VERSION)
-# - libsqlite3.so.0 =symlink-> libsqlite3.so.$(PACKAGE_VERSION) (see below)
-# - libsqlite3.so   =symlink-> libsqlite3.so.3
+# - libsqlite3.so.0      =symlink-> libsqlite3.so.$(PACKAGE_VERSION) (see below)
+# - libsqlite3.so        =symlink-> libsqlite3.so.3
+#
+# N.B. we initially had a link named libsqlite3.so.3 but it's
+# unnecessary unless we want to set SONAME to libsqlite3.so.3, which
+# is also unnecessary.
 #
 # The link named libsqlite3.so.0 is provided in an attempt to reduce
 # downstream disruption when performing upgrades from pre-3.48 to a
@@ -1429,7 +1453,7 @@ all: so
 #    down-side of this is that it may upset packaging tools when we
 #    replace libsqlite3.so (from a legacy package) with a new symlink.
 #
-# 2) If INSTALL_SO_086_LINKS=1 and point (1) does not apply then links
+# 2) If INSTALL_SO_086_LINK=1 and point (1) does not apply then links
 #    to the legacy-style names are created. The primary intent of this
 #    is to enable chains of operations such as the hypothetical (apt
 #    remove sqlite3-3.47.0 && apt install sqlite3-3.48.0). In such
@@ -1443,10 +1467,9 @@ install-so-1: $(install-dir.lib) $(libsqlite3.SO)
 	$(INSTALL) $(libsqlite3.SO) "$(install-dir.lib)"
 	@echo "Setting up $(libsqlite3.SO) symlinks..."; \
 		cd "$(install-dir.lib)" || exit $$?; \
-		rm -f $(libsqlite3.SO).3 $(libsqlite3.SO).0 $(libsqlite3.SO).$(PACKAGE_VERSION) || exit $$?; \
+		rm -f $(libsqlite3.SO).0 $(libsqlite3.SO).$(PACKAGE_VERSION) || exit $$?; \
 		mv $(libsqlite3.SO) $(libsqlite3.SO).$(PACKAGE_VERSION) || exit $$?; \
 		ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO) || exit $$?; \
-		ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).3 || exit $$?; \
 		ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0 || exit $$?; \
 		ls -la $(libsqlite3.SO) $(libsqlite3.SO).[03]*; \
 		if [ -e $(libsqlite3.SO).0.8.6 ]; then \
@@ -1454,8 +1477,8 @@ install-so-1: $(install-dir.lib) $(libsqlite3.SO)
 			rm -f libsqlite3.la $(libsqlite3.SO).0.8.6 || exit $$?; \
 			ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0.8.6 || exit $$?; \
 			ls -la $(libsqlite3.SO).0.8.6; \
-		elif [ x1 = "x$(INSTALL_SO_086_LINKS)" ]; then \
-			echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINKS=1"; \
+		elif [ x1 = "x$(INSTALL_SO_086_LINK)" ]; then \
+			echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINK=1"; \
 			rm -f libsqlite3.la $(libsqlite3.SO).0.8.6 || exit $$?; \
 			ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0.8.6 || exit $$?; \
 			ls -la $(libsqlite3.SO).0.8.6; \
@@ -1536,7 +1559,7 @@ tclextension: tclsqlite3.c
 # to find it.
 #
 tclextension-install: tclsqlite3.c
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --cc "$(T.cc)" $(CFLAGS.tclextension)
+	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --destdir "$(DESTDIR)" --cc "$(T.cc)" $(CFLAGS.tclextension)

 #
 # Uninstall the SQLite TCL extension that is used by $TCLSH_CMD.
@@ -1578,7 +1601,7 @@ fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon$(B.exe)

 fts5parse.h: fts5parse.c

-fts5.c: $(FTS5_SRC) $(B.tclsh) # has_tclsh84
+fts5.c: $(FTS5_SRC) $(B.tclsh)
 	$(B.tclsh) $(TOP)/ext/fts5/tool/mkfts5c.tcl
 	cp $(TOP)/ext/fts5/fts5.h .

@@ -1619,7 +1642,7 @@ testfixture$(T.exe):	$(T.tcl.env.sh) has_tclsh85 $(TESTFIXTURE_SRC)
 	$(T.link.tcl) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) \
 		$$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC \
-		$(CFLAGS.libsqlite3) $(LDFLAGS.libsqlite3)
+		$(LDFLAGS.libsqlite3)

 coretestprogs:	testfixture$(B.exe) sqlite3$(B.exe)

@@ -1731,22 +1754,43 @@ smoketest:	$(TESTPROGS) fuzzcheck$(T.exe)
 shelltest:
 	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl release shell

+#
+# sqlite3_analyzer.c build depends on $(LINK_TOOLS_DYNAMICALLY).
+#
+sqlite3_analyzer.c.flags.0 = -DINCLUDE_SQLITE3_C=1
+sqlite3_analyzer.c.flags.1 =
 sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl \
-                    $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
-	$(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
+                    $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
+	$(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in \
+		$(sqlite3_analyzer.c.flags.$(LINK_TOOLS_DYNAMICALLY)) \
+		$(OPT_FEATURE_FLAGS) \
+		> $@

-sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c
-	$(T.link.tcl) sqlite3_analyzer.c -o $@ $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC \
-		$(LDFLAGS.libsqlite3)
+#
+# sqlite3_analyzer's build mode depends on $(LINK_TOOLS_DYNAMICALLY).
+#
+sqlite3_analyzer.flags.1 = -L. -lsqlite3
+sqlite3_analyzer.flags.0 = $(LDFLAGS.libsqlite3)
+sqlite3_analyzer.deps.1 = $(libsqlite3.SO)
+sqlite3_analyzer.deps.0 =
+sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c \
+                          $(sqlite3_analyzer.deps.$(LINK_TOOLS_DYNAMICALLY))
+	$(T.link.tcl) sqlite3_analyzer.c -o $@ \
+		$(sqlite3_analyzer.flags.$(LINK_TOOLS_DYNAMICALLY)) \
+		$$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC $$TCL_LIBS
+# ^^^^ the order of those flags is relevant for
+# $(sqlite3_analyzer.flags.1): if the $$TCL_... flags come first they
+# can cause the $@ to link to an out-of-tree libsqlite3.so, which may
+# or may not fail or otherwise cause confusion.

 sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl \
            $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl \
-            $(TOP)/tool/sqltclsh.c.in has_tclsh85
+            $(TOP)/tool/sqltclsh.c.in
 	$(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c

 sqltclsh$(T.exe): $(T.tcl.env.sh) sqltclsh.c
-	$(T.link.tcl) sqltclsh.c -o $@ $$TCL_INCLUDE_SPEC $(CFLAGS.libsqlite3) \
-		$$TCL_LIB_SPEC $(LDFLAGS.libsqlite3)
+	$(T.link.tcl) sqltclsh.c -o $@ $$TCL_INCLUDE_SPEC \
+		$(LDFLAGS.libsqlite3) $$TCL_LIB_SPEC $$TCL_LIBS
 # xbin: target for generic binaries which aren't usually built. It is
 # used primarily for testing the build process.
 xbin: sqltclsh$(T.exe) sqlite3_analyzer$(T.exe)
@@ -1767,12 +1811,12 @@ CHECKER_DEPS =\
  $(TOP)/ext/misc/btreeinfo.c \
  $(TOP)/ext/repair/sqlite3_checker.c.in

-sqlite3_checker.c:	$(CHECKER_DEPS) has_tclsh85
+sqlite3_checker.c:	$(CHECKER_DEPS)
 	$(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@

 sqlite3_checker$(T.exe):	$(T.tcl.env.sh) sqlite3_checker.c
 	$(T.link.tcl) sqlite3_checker.c -o $@ $$TCL_INCLUDE_SPEC \
-		$(CFLAGS.libsqlite3) $$TCL_LIB_SPEC $(LDFLAGS.libsqlite3)
+		$$TCL_LIB_SPEC $(LDFLAGS.libsqlite3)
 xbin: sqlite3_checker$(T.exe)

 dbdump$(T.exe): $(TOP)/ext/misc/dbdump.c sqlite3.o
@@ -1919,15 +1963,27 @@ threadtest5: sqlite3.c $(TOP)/test/threadtest5.c
 	$(T.link) $(TOP)/test/threadtest5.c sqlite3.c -o $@ $(LDFLAGS.libsqlite3)
 xbin: threadtest5

-# The standard CLI is built using the amalgamation since it uses
-# special compile-time options that are interpreted by individual
-# source files within the amalgamation.
+#
+# When building sqlite3$(T.exe) we specifically embed a copy of
+# sqlite3.c, and not link to libsqlite3.so or libsqlite3.a, because
+# the shell needs to be able to enable arbitrary library features,
+# some of which have significant performance impacts. For example,,
+# SQLITE_ENABLE_EXPLAIN_COMMENTS has been measured as having a 5.2%
+# runtime performance hit, which is fine for use in the shell but is
+# not appropriate for the canonical library build.
 #
 sqlite3$(T.exe):	shell.c sqlite3.c
 	$(T.link) -o $@ \
 		shell.c sqlite3.c \
 		$(CFLAGS.readline) $(SHELL_OPT) $(CFLAGS.icu) \
 		$(LDFLAGS.libsqlite3) $(LDFLAGS.readline)
+#
+# Build sqlite3$(T.exe) by default except in wasi-sdk builds.  Yes, the
+# semantics of 0 and 1 are confusingly swapped here.
+#
+sqlite3$(T.exe)-1:
+sqlite3$(T.exe)-0: sqlite3$(T.exe)
+all: sqlite3$(T.exe)-$(HAVE_WASI_SDK)

 # The "sqlite3d" CLI is build using separate source files.  This
 # is useful during development and debugging.
@@ -1938,21 +1994,19 @@ sqlite3d$(T.exe):	shell.c $(LIBOBJS0)
 		$(CFLAGS.readline) $(SHELL_OPT) \
 		$(LDFLAGS.libsqlite3) $(LDFLAGS.readline)

-#
-# Build sqlite3$(T.exe) by default except in wasi-sdk builds.  Yes, the
-# semantics of 0 and 1 are confusingly swapped here.
-#
-sqlite3$(T.exe)-1:
-sqlite3$(T.exe)-0 sqlite3$(T.exe)-: sqlite3$(T.exe)
-all: sqlite3$(T.exe)-$(HAVE_WASI_SDK)
-
 install-shell-0: sqlite3$(T.exe) $(install-dir.bin)
 	$(INSTALL) -s sqlite3$(T.exe) "$(install-dir.bin)"
-install-shell-1 install-shell-:
+install-shell-1:
 install: install-shell-$(HAVE_WASI_SDK)

-sqldiff$(T.exe):	$(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.o sqlite3.h
-	$(T.link) -o $@ $(TOP)/tool/sqldiff.c sqlite3.o $(LDFLAGS.libsqlite3)
+# How to build sqldiff$(T.exe) depends on $(LINK_TOOLS_DYNAMICALLY)
+#
+sqldiff.0.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.o sqlite3.h
+sqldiff.0.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c sqlite3.o $(LDFLAGS.libsqlite3)
+sqldiff.1.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h $(libsqlite3.SO)
+sqldiff.1.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c -L. -lsqlite3 $(LDFLAGS.configure)
+sqldiff$(T.exe): $(sqldiff.$(LINK_TOOLS_DYNAMICALLY).deps)
+	$(sqldiff.$(LINK_TOOLS_DYNAMICALLY).rules)

 install-diff: sqldiff$(T.exe) $(install-dir.bin)
 	$(INSTALL) -s sqldiff$(T.exe) "$(install-dir.bin)"
@@ -2133,7 +2187,7 @@ SHELL_DEP = \
    $(TOP)/src/test_windirent.c \
    $(TOP)/src/test_windirent.h

-shell.c:	$(SHELL_DEP) $(TOP)/tool/mkshellc.tcl $(B.tclsh) # has_tclsh84
+shell.c:	$(SHELL_DEP) $(TOP)/tool/mkshellc.tcl $(B.tclsh)
 	$(B.tclsh) $(TOP)/tool/mkshellc.tcl >shell.c

 #
@@ -2201,7 +2255,7 @@ sqlite3.def: $(LIBOBJ)
 		| sed 's/^.* _//' >>sqlite3.def

 sqlite3.dll: $(LIBOBJ) sqlite3.def
-	$(T.cc.sqlite) $(LDFLAGS.shobj) -o $@ sqlite3.def \
+	$(T.cc.sqlite) $(LDFLAGS.shlib) -o $@ sqlite3.def \
 		-Wl,"--strip-all" $(LIBOBJ) $(LDFLAGS.configure)

 #
@@ -1,9 +1,9 @@
-C Enhance\sthe\svfstrace\sextension\ssuch\sthat\sthe\soutput\scan\sbe\scontrolled\susing\nthe\s"PRAGMA\svfstrace('...');"\sstatement.\s\sSee\sheader\scomment\son\sthe\ssource\scode\nfor\sdetails.
-D 2024-11-15T20:39:41.451
+C Simplification\sto\sthe\sfix\sin\sthe\sprior\scheck-in.
+D 2024-11-28T02:09:27.502
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
-F Makefile.in 2c90ab3183f810086878a784c323b7816238a5e6943d267c25a71edc623a05a3
+F Makefile.in 85f2c740cadf969abbd9a6210b8b76636dbf231b9d3efe977b060c3055fac5b9
 F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0
 F Makefile.msc a92237976eb92c5efaa0dd2524746aec12c196e12df8d4dbff9543a4648c3312
 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159
@@ -13,7 +13,7 @@ F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
-F auto.def ec2fa2ea6fd691af4a991e1906da314ce094d019b1b9d08563167b138114bbdc
+F auto.def 961d2cf22b65d31f9eb97470b50cbbdca34b999cf86f438811c1ec214b296b8b
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am adedc1324b6a87fdd1265ddd336d2fb7d4f36a0e77b86ea553ae7cc4ea239347
 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac
@@ -37,8 +37,8 @@ F autoconf/tea/win/rules.vc 94a18c3e453535459b4a643983acca52fb8756e79055bd2ad4b0
 F autoconf/tea/win/targets.vc 96a25a1fa6e9e9cfb348fd3760a5395b4ce8acafc8ed10f0412937ec200d5dbd
 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4
 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e
-F autosetup/README.md f12fd1556b50ff33ebf5c1476f4640c6010b3093ba8d75ba12b5417884011d5e
-F autosetup/autosetup 9416ffdcdd6e2dbf7f6d1e5c890078518930f8af7722a950eacc28c7f151d2d6 x
+F autosetup/README.md 083b4417637f134875709dfa8ca149ce9b3634725b65e0f7d9b96f2bf56caa3a
+F autosetup/autosetup df8b53928b1fe3c67db5bc77c8e1eb8160c1b6a26c370e9a06c68748f803b7e4 x
 F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x
 F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x
 F autosetup/autosetup-find-tclsh 25905f6c302959db80c2951aa267b4411c5645b598ce761cfc24a166141e2c4c x
@@ -49,7 +49,7 @@ F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1d
 F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f
 F autosetup/jimsh0.c d40e381ea4526a067590e7b91bd4b2efa6d4980d286f908054c647b3df4aee14
 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba
-F autosetup/proj.tcl dd4cdf0c31967056bc2c1956b0601118182b7e143cffb184d79c6a625ae9ef87
+F autosetup/proj.tcl 22556a325c964aa5377d4d881722385f41fcd7c1b60102ba8965f7814c83e9ce
 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9
 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
@@ -69,7 +69,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4
 F ext/expert/expert1.test 1d2da6606623b57bb47064e02140823ce1daecd4cacbf402c73ad3473d7f000c
-F ext/expert/sqlite3expert.c 9d87c5eeb86707e4dbf140ca20a32935f88cfb5d8da94a406b7e0f0cdb815af6
+F ext/expert/sqlite3expert.c 494a6b7d4e0ead6dec6a50109dd78fcc054bb1a3fcc29c6f25e06a3685ed557e
 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b
 F ext/expert/test_expert.c b767b2039a0df707eb3147e86bcf68b252d8455d9a41774b1a836cd052ceca70
 F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c7cc3bf59ee
@@ -80,7 +80,7 @@ F ext/fts3/fts3.c 9f8ce82bbf4ec0636e6170e58f17b04817fa4c39b2d5126ac06f005d485f6d
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 968f7d7cae541a6926146e9fd3fb2b2ccbd3845b7890a8ed03de0c06ac776682
 F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3
-F ext/fts3/fts3_expr.c 903bfb9433109fffb10e910d7066c49cbf8eeae316adc93f0499c4da7dfc932a
+F ext/fts3/fts3_expr.c 365849a2a1185e19028a9db2d9f1ea63efe909a3a6aca7ec86fc26a13a60bd58
 F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
 F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116
@@ -103,20 +103,20 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15
-F ext/fts5/fts5.h 6b4b92df890965567360db5f1ead24fd13a72cb23b95e4ed2ff58d1d89f7aa42
-F ext/fts5/fts5Int.h bf0d3efa144f36e00f9b5206626aec2f436f58186a0835092394f2202e9828e3
+F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2
+F ext/fts5/fts5Int.h 6abff7dd770dc5969c994c281e6e77fc277ce414d56cc4a62c145cc7036b0b67
 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759
 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
-F ext/fts5/fts5_config.c a6633d88596758941c625b526075b85d3d9fd1089d8d9eab5db6e8a71fd347ad
-F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33
+F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8
+F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c 368a968570ce12ba40223e284a588d9f93ee23a0133727f0df1fcd64086b1fb6
-F ext/fts5/fts5_main.c 0b60324001e26d79e8a12e544883792f59a58717f6dce9fb1f8878f0e0196530
+F ext/fts5/fts5_index.c cef6791bd9f9db4305494292d6dd5d24a7379aabf370a4d6b559e16b740fa88e
+F ext/fts5/fts5_main.c a4f6a46e5ede02d40fc23655d3de37e55f75210ae3856b2b518a2295d3331543
 F ext/fts5/fts5_storage.c 337b05e4c66fc822d031e264d65bde807ec2fab08665ca2cc8aaf9c5fa06792c
 F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
-F ext/fts5/fts5_tokenize.c 87ab719f0556172da3414f1741c11bb4d333ebecde157945a55478bfe6e46c44
+F ext/fts5/fts5_tokenize.c 49aea8cc400a690a6c4f83c4cedc67f4f8830c6789c4ee343404f62bcaebca7b
 F ext/fts5/fts5_unicode2.c 6f9b0fb79a8facaed76628ffd4eb9c16d7f2b84b52872784f617cf3422a9b043
 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
 F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0
@@ -212,11 +212,12 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47
 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed
 F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3
-F ext/fts5/test/fts5origintext.test 2015f69bc8abd111152a8e66211fd2d45026378001e07c054159aa4f84e6691d
+F ext/fts5/test/fts5origintext.test 3b73aa036ce5244bb7c5782c5441b979585bdca026accf75d16026a2a8119c09
 F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba
-F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b
+F ext/fts5/test/fts5origintext3.test 4988b6375acc3bbb0515667765f57e389caf449814af9c1095c053f7de2b4223
 F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8
 F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b
+F ext/fts5/test/fts5origintext6.test 09eb1347cb0dceaebbebf3d3e6bd5d24c7c1006efddc2984540450324bbdafa4
 F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c
 F ext/fts5/test/fts5plan.test f8b0d752a818059a934cdc96c0f77de058a67a0a57bb3a8181d28307ab5b1626
 F ext/fts5/test/fts5porter.test 15b514fac8690b58e99c330efe5bf5615bc43f2fae4a3cca3f923dbaff55a0c0
@@ -245,6 +246,7 @@ F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988
 F ext/fts5/test/fts5synonym2.test 58f357b997cf2fedeeb9d0de4db9f880fa96fa2fe27a743bfe7d7b96895bdd87
 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
+F ext/fts5/test/fts5tokendata.test 7cad79af82e8e81b7a36b450087233d2fca051bb0d584421afc375d6dd26d6f6
 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98
 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6
 F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3
@@ -690,13 +692,13 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
 F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
 F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
-F ext/wasm/tester1.c-pp.js b683e64f776e03dc7cb81cf4737b991c644b0a59e3ca52dcdc606d851a95475e
+F ext/wasm/tester1.c-pp.js 228101c290003423f0bfb66a6ebbfc6904fa7b1b69466e700c135f74ee83d62a
 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e
 F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
 F ext/wasm/wasmfs.make bc8bb227f35d5bd3863a7bd2233437c37472a0d81585979f058f9b9b503bef35
 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0
-F main.mk 579c3f70be718c97b82c12c14942305ed0c330619355aad26ed14a752f6daf57
+F main.mk 1ec3c7a81198397b14bf75c9304f01e891a5d464797cd402d4f3781feadd3331
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421
@@ -724,10 +726,10 @@ F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a
 F src/dbpage.c db1be8adaf1f839ad733c08baeac5c22aa912f7b535865c0c061382602081360
 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c
 F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42
-F src/expr.c a9d9f5fdfbdd3b2c94d7af1b11f181464b8a641736cf32cb92fa3c5e7ecb30df
+F src/expr.c b838969c58a38dfedabec18c432204895c7333d7a509a20fa63c39bf92001f0e
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f
-F src/func.c fa138d44348e189817542f6efa6232420b3e0081c835ced65883adc7fd777d65
+F src/func.c 49489dcce46d2d491cedb451e974264150c473e5f5bba448498a9aa4c1993537
 F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b
 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
@@ -737,7 +739,7 @@ F src/insert.c f8d1a0f8ee258411009c6b7f2d93170e351bd19f5ad89d57e1180644297cbe70
 F src/json.c 68a98c020c22127f2d65f08855f7fc7460ff352a6ce0b543d8931dde83319c22
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
-F src/main.c 9f4286302727f58fddc03a820d24cb7618a1e27473501792fbe979726f846d1f
+F src/main.c efacea3a809ba236233c4bc7648e623c28cb5fd1e1a343e74fe772005f142d8c
 F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -759,23 +761,23 @@ F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e
 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a
 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107
 F src/os_unix.c d2edbd92b07a3f778c2defa8a2e9d75acceb6267bda56948c41e8cdda65224d6
-F src/os_win.c db4baa8f62bbfe3967c71b008cea31a8f2ff337c1667ff4d8a677e697315ff0d
+F src/os_win.c 49c7725b500f5867e8360e75eeb30f9d70b62fa1f05c8a101da627210578df32
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 9656ad4e8331efb8a4f94f7a0c6440b98caea073950a367ea0c728a53b8e62c9
 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a
-F src/parse.y 8ec56598aa0df92428627502267d0d1c9778cc27308f8ffd31dfb2d017a8755f
+F src/parse.y dcf45a81b61223ac93e61fdfe9b22d635dd371c446e8222634d90aa37e25e5f6
 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319
 F src/pragma.c a2ec3657a953fa7dea7c1e680e4358b6ce6ae570b6c5234e0f5ef219d308d223
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c 1832be043fce7d489959aae6f994c452d023914714c4d5457beaed51c0f3d126
-F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2
+F src/printf.c 96f7f8baeedc7639da94e4e7a4a2c200e2537c4eec9e5e1c2ffc821f40eb3105
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c c8a5372b97b2a2e972a280676f06ddb5b74e885d3b1f5ce383f839907b57ef68
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe
-F src/shell.c.in c61d7af58e678d479f1450e4ea236d2d6ac3bb16a636030104f210303b30d563
+F src/shell.c.in 7aa68b305246391984c48f10725416079394a64bf0349fa4087d835c41ef733d
 F src/sqlite.h.in 4d93768709c53b7c653a63817a82d5a8625264ca0d8cdf99967ba147bdcf2aa6
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
@@ -783,7 +785,7 @@ F src/sqliteInt.h 77be043f8694f4a8702d0ee882022b2e5a6489a0493e77c5d9a73f1efc5a2c
 F src/sqliteLimit.h 6993c9cfe3af5b8169ae0e5f15627fc15596726d4f1dc90a221309f79715ce88
 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
-F src/tclsqlite.c ff2dc3ec1bd318ee7a45d6b246a367703d5fb2a4c8da99d675ee7eb987b3a153
+F src/tclsqlite.c 1c2f697cb12a1d49f5e0b448327f7cf614809423bb43753b2d97f87354298113
 F src/tclsqlite.h 65e2c761446e1c9fa0342b7d2612a703483643c8b6a316d12a65b745a4727395
 F src/test1.c 2d507751bfb4aa254dc22588ef1e3c5c5cfcb2e636d0e6e1fa0bbd307669c2a8
 F src/test2.c 7ebc518e6735939d8979273a6f7b1d9b5702babf059f6ad62499f7f60a9eb9a3
@@ -804,7 +806,7 @@ F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c9
 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86
 F src/test_fs.c c411c40baba679536fc34e2679349f59d8225570aed3488b5b3ef1908525a3d5
 F src/test_func.c 8c0e89192f70fac307822d1ac2911ee51751288780b3db0c5ab5ca75fa0fe851
-F src/test_hexio.c 0f777bf9fbb2684bb4978372bacc90ef7337d5d9e3cebe067a9409941e88bacf
+F src/test_hexio.c 7449504e4bde876ba91b202617a9228c7c8c2e7bd8b957302f3803ac0e9e353c
 F src/test_init.c 17313332d58e90defc527129d5eda4a08bd6b6e8de7207a231523c8d98fb445e
 F src/test_intarray.c e4216aadee9df2de7d1aee7e70f6b22c80ee79ece72a63d57105db74217639e5
 F src/test_intarray.h 6c3534641108cd1bea517a8e117dcba237081310a29a4c35bd2190caa8972293
@@ -836,7 +838,7 @@ F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68
-F src/treeview.c 88aa39b754f5ef7214385c1bbbdd2f3dc20efafeed0cf590e8d1199b9c6e44aa
+F src/treeview.c 4eeb155abefd88a60d0c37cc00bcfac38a8dd566970f019e4af7e02672ee2599
 F src/trigger.c 0bb986a5b96047fd597c6aac28588853df56064e576e6b81ba777ef2ccaac461
 F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508
 F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1
@@ -858,9 +860,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 8b7e309a8012659ac9275ad8cdcc6acaf73fa04b1090e38a01335f230fd10681
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
-F src/where.c 4de9e7ca5f49e4a21c1d733e2b2fbbc8b62b1a157a58a562c569da84cfcb005b
+F src/where.c b34adb09fdb2a4f57a0bafd4194b501047ef383976fef9f0ac3f395f4881694d
 F src/whereInt.h 1e36ec50392f7cc3d93d1152d4338064cd522b87156a0739388b7e273735f0ca
-F src/wherecode.c 81b9af89f4f85c8097d0da6a31499f015eabc4d3584963d30ba7b7b782e26514
+F src/wherecode.c 0c3d3199a2b769a5e2bb70feb5003dc85b3d86842ecaf903a47f2b4205ca5dab
 F src/whereexpr.c 0f93a29cabd3a338d09a1f5c6770620a1ac51ec1157f3229502a7e7767c60b6f
 F src/window.c 6c386af5972a58f9a9847bba9d7ca70c4c682391ab8478d94a6e046b22a0dbb3
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1041,7 +1043,7 @@ F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4
 F test/corruptK.test ac13504593d89d69690d45479547616ed12644d42b5cb7eeb2e759a76fc23dcb
 F test/corruptL.test 652fc8ac0763a6fd3eb28b951d481924167b2d9936083bcc68253b2274a0c8fe
 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067
-F test/corruptN.test 40bc47aee4af9aadff902be43f14d69dc17b3731448dad6c7cc722da913f1455
+F test/corruptN.test a034bb217bebd8d007625dfb078e76ec3d53515052dbceb68bd47b2c27674d5c
 F test/cost.test cc434a026b1e9d0d98137a147e24e5daf1b1ad09e9ff7da63b34c83ddd136d92
 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249
 F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc
@@ -1100,7 +1102,7 @@ F test/e_createtable.test 31b9bcb6ac8876bc7ec342d86d9c231a84c62b442093a6651dfd0f
 F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e
 F test/e_droptrigger.test 235c610f8bf8ec44513e222b9085c7e49fad65ad0c1975ac2577109dd06fd8fa
 F test/e_dropview.test 74e405df7fa0f762e0c9445b166fe03955856532e2bb234c372f7c51228d75e7
-F test/e_expr.test b950818a48269506d75a41c819003bd77a0893bc4a4f2fdee191bc74109c1a87
+F test/e_expr.test 4faef475076f676e2a009270dbd3a7658db30d12dc3d21e4b85cab5f50be18c5
 F test/e_fkey.test feeba6238aeff9d809fb6236b351da8df4ae9bda89e088e54526b31a0cbfeec5
 F test/e_fts3.test 17ba7c373aba4d4f5696ba147ee23fd1a1ef70782af050e03e262ca187c5ee07
 F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e
@@ -1124,6 +1126,7 @@ F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020
 F test/eqp.test 82f221e8cd588434d7f3bba9a0f4c78cbe7a541615a41632e12f50608bfb4a99
 F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd
 F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9
+F test/errofst1.test 6da78363739ba8991f498396ab331b5d64e7ab5c4172c12b5884683ef523ac53
 F test/eval.test 73969a2d43a511bf44080c44485a8c4d796b6a4f038d19e491867081155692c0
 F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650bf0747
 F test/exclusive2.test cd70b1d9c6fffd336f9795b711dcc5d9ceba133ad3f7001da3fda63615bdc91e
@@ -1204,7 +1207,7 @@ F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8
 F test/fts3expr4.test 6c7675bbdbffe6ffc95e9e861500b8ac3f739c4d004ffda812f138eeb1b45529
 F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1cf983e
 F test/fts3f.test 8c438d5e1cab526b0021988fb1dc70cf3597b006a33ffd6c955ee89929077fe3
-F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45c5df45
+F test/fts3fault.test 9228f00cd69e2a5d2ed0f06c181981f4f90bd36da9f86b73f3a58b4b23451fd4
 F test/fts3fault2.test 7b2741e5095367238380b0fcdb837f36c24484c7a5f353659b387df63cf039ec
 F test/fts3fault3.test ccdd2292dd2d4e21e30fc5f4c8e064f79e516087eec5ff57ab6bc4f6a7714097
 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
@@ -1300,7 +1303,7 @@ F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
 F test/in4.test bb767ec1cfd1730256f0a83219f0acda36bc251b63f8b8bb7d8c7cff17875a4f
 F test/in5.test 4fd79c70dfa0681313e8cdca07f5ff0400bdc0e20f808a5c59eaef1e4b48082a
 F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b
-F test/in7.test 9256cdb30dc487f2078bb4bb30f43f2c1ff4d277a9c7c9a14bd1c9510c9c8cae
+F test/in7.test 5050b648510d88bd27ff6b40991a45e1cc277c20e258162e81650e01069a56bb
 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822
 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f
 F test/incrblob3.test 67621a04b3084113bf38ce03797d70eca012d9d8f948193b8f655df577b0da6f
@@ -1377,7 +1380,7 @@ F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb9
 F test/json/json-speed-check.sh 912ee03e700a65c827ee0c7b4752c21ec5ef69cf7679d2f482ca817042bead52 x
 F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7
 F test/json101.test 30db5b055b103ccabc53a29cfe6cda3345d07e171aeb25403dafa04f19e98b19
-F test/json102.test 4b3a0f94535f033239b67c13dbee8b47d2b5ee467e0f2fdab5eadf370bbe5fd3
+F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1
 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
 F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988
@@ -1583,7 +1586,7 @@ F test/savepoint.test 6e9804a17767f08432c7a5e738b9a8f4b891d243110b63d3a41d270d3d
 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7
 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0
 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
-F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
+F test/savepoint6.test 48a645a7bb3a59a6fcf06a7364cfe5b655c336760de39068f7c241b0fc80d963
 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa
 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2
 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e
@@ -1694,14 +1697,14 @@ F test/subquery.test 903abf41049f8404256f7be24b3151328304a5b25162e17ab0079460237
 F test/subquery2.test 90cf944b9de8204569cf656028391e4af1ccc8c0cc02d4ef38ee3be8de1ffb12
 F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303
 F test/substr.test a673e3763e247e9b5e497a6cacbaf3da2bd8ec8921c0677145c109f2e633f36b
-F test/subtype1.test 7a9c55ed84d4ce551203d18046f925e293d75f69da81bff71aaf2696e4a2a748
+F test/subtype1.test 96fd2a59bfc845c955b5f339d23b37ef4d50de5f8a04acd1450a68605fa2e3e7
 F test/superlock.test 85256830339a6871ce36a2ef591c3f67716a701b5497788fb2068b90159c2442
 F test/swarmvtab.test 250231404fcac88f61a6c147bb0e3a118ed879278cd3ccb0ae2d3a729e1e8e26
 F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c
 F test/swarmvtab3.test 41a3ab47cb7a834d4e5336425103b617410a67bb95d335ef536f887587ece073
 F test/swarmvtabfault.test 8a67a9f27c61073a47990829e92bc0c64420a807cb642b15a25f6c788210ed95
 F test/symlink.test 4368af0e213dd6e726a6240a16f2bb96a5a58f83f2d5d60652f27547b28cbf06
-F test/symlink2.test 9531f475a53d8781c4f81373f87faf2e2aff4f5fb2102ec6386e0c827916a670
+F test/symlink2.test bf932ff7fe95c9dbb39d2a990df9098b0ea943233c97e40098e0a8d6b559a96f
 F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d4333092
 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039
 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37
@@ -1719,7 +1722,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
 F test/tester.tcl 7b44f1a9b9a2de8112695b908afc21dd9a68cd2d44e84b73f1b27b53492c0d59
-F test/testrunner.tcl c40d5700578f8c9d00e0e15f105f645c471bc2c48eb8b013bd2953400f2c6bf0 x
+F test/testrunner.tcl 90ed8b6c2b26dc1f6af08aeb04670a5df86172f3d9828d8af000f972afa50061 x
 F test/testrunner_data.tcl ba4aeea28aa03cfa6fe7e57782ddecb7a7b91c3a0b3251583cb4f0ee002de6a6
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502
@@ -1986,7 +1989,7 @@ F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395
 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12
 F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c
 F test/wal.test 519c550255c78f55959e9159b93ebbfad2b4e9f36f5b76284da41f572f9d27da
-F test/wal2.test 44fe1cb4935dbbddfa0a34c2c4fd90f0ba8654d59b83c4136eb90fb327fd264f
+F test/wal2.test e89ca97593b5e92849039f6b68ce1719a853ef20fa22c669ec1ac452fbc31cab
 F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1995,7 +1998,7 @@ F test/wal64k.test 2a525c0f45d709bae3765c71045ccec5df7d100ccbd3a7860fdba46c9addb
 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd
 F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8
 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750
-F test/wal_common.tcl 4589f701d5527ace2eba43823c96c2177e1f9dd2a6098256ee2203a0a313c13a
+F test/wal_common.tcl 204d1721ac13c5e0c7fae6380315b5ab7f4e8423f580d826c5e9df1995cb018d
 F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946
 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434
 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d
@@ -2105,7 +2108,7 @@ F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e045
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
 F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x
 F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f
-F tool/buildtclext.tcl f046701e0bfa7c70116555dd4c3fbd7a8b61779c10a9e7586936c77a118c4b09
+F tool/buildtclext.tcl 5e1f1aa843e635c8b7480c7d1ec1f149a5e52136ae2fca1226304053a1a60587
 F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x
 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
 F tool/cktclsh.sh 6075eef9c6b9ba4b38fef2ca2a66d25f2311bd3c610498d18a9b01f861629cca
@@ -2135,7 +2138,7 @@ F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669
 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176
 F tool/mkautoconfamal.sh cbdcf993fa83dccbef7fb77b39cdeb31ef9f77d9d88c9e343b58d35ca3898a6a
-F tool/mkccode.tcl 4cb8ad7e7330aaed052b0657a1bfacbc67103c400e41860aff643a482cfc2d3e x
+F tool/mkccode.tcl 210159febe0ef0ecbc53c79833500663ceaba0115b2b374405818dc835b5f84b x
 F tool/mkctimec.tcl ef6a67ec82e5b6fc19152a4c79f237227b18bf67ff16d155bac7adb94355d9cf x
 F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559
 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a
@@ -2178,7 +2181,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x
 F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60
 F tool/sqldiff.c 2a0987d183027c795ced13d6749061c1d2f38e24eddb428f56fa64c3a8f51e4b
-F tool/sqlite3_analyzer.c.in 348ba349bbdc93c9866439f9f935d7284866a2a4e6898bc906ae1204ade56918
+F tool/sqlite3_analyzer.c.in fc7735c499d226a49d843d8209b2543e4e5229eeb71a674c331323a2217b65b4
 F tool/sqlite3_rsync.c 9a1cca2ab1271c59b37a6493c15dc1bcd0ab9149197a9125926bc08dd26b83fb
 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898
 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848
@@ -2189,8 +2192,8 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/stripccomments.c 20b8aabc4694d0d4af5566e42da1f1a03aff057689370326e9269a9ddcffdc37
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
 F tool/symbols.sh 1612bd947750e21e7b47befad5f6b3825b06cce0705441f903bf35ced65ae9b9
+F tool/tclConfigShToAutoDef.sh 44ec55046d86a3febb2cb3e099399b41794e80e9cd138eee7b9b016f819e882b x
 F tool/tclConfigShToMake.sh 7c065d81c2d178e15e45a77372c6e5a38b5a1b08755301cd6f20a3a862db7312 x
-F tool/tclConfigShToTcl.sh 44ec55046d86a3febb2cb3e099399b41794e80e9cd138eee7b9b016f819e882b x
 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003
 F tool/vdbe-compress.tcl fa2f37ab39b2a0087fafb6a7f3ce19503e25e624ffa8ed9951717ab72920c088
 F tool/vdbe_profile.tcl 3ac5a4a9449f4baf77059358ea050db3e34395ccf59c5464d29b91746d5b961e
@@ -2198,8 +2201,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 0773677b553e032e992266c6c75e10565729238df3ef52b56602a92cf651bea7
-R 1cea5b1a16b6733cdd9ac2dd470ae319
+P df95d5209a77c0302e08ea7b1f248646fb2a062ba5beca898678aee0a1d03a3a
+R 2a463ba8c7e54a6e85c2163c1e1ce03b
 U drh
-Z 6b0f9afcc9547572803b9a01d3227a70
+Z 5b8128e8d052469f340725fed84274b9
 # Remove this line to create a well-formed Fossil manifest.
@@ -1 +1 @@
-96105d33597765c23dbd490b3aa0c2273731d1970d7041720e9f043dbe3517b3
+f79cb748c9447ca4d2184a24219feb19571eb79870e0bedf14704743a3d5f8b9
@@ -6180,16 +6180,23 @@ void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){
 ** same as that currently bound to variable pVar, non-zero is returned.
 ** Otherwise, if the values are not the same or if pExpr is not a simple
 ** SQL value, zero is returned.
+**
+** If the SQLITE_EnableQPSG flag is set on the database connection, then
+** this routine always returns false.
 */
-static int exprCompareVariable(
+static SQLITE_NOINLINE int exprCompareVariable(
  const Parse *pParse,
  const Expr *pVar,
  const Expr *pExpr
 ){
-  int res = 0;
+  int res = 2;
  int iVar;
  sqlite3_value *pL, *pR = 0;
 
+  if( pExpr->op==TK_VARIABLE && pVar->iColumn==pExpr->iColumn ){
+    return 0;
+  }
+  if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) return 2;
  sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, SQLITE_AFF_BLOB, &pR);
  if( pR ){
    iVar = pVar->iColumn;
@@ -6199,12 +6206,11 @@ static int exprCompareVariable(
      if( sqlite3_value_type(pL)==SQLITE_TEXT ){
        sqlite3_value_text(pL); /* Make sure the encoding is UTF-8 */
      }
-      res =  0==sqlite3MemCompare(pL, pR, 0);
+      res = sqlite3MemCompare(pL, pR, 0) ? 2 : 0;
    }
    sqlite3ValueFree(pR);
    sqlite3ValueFree(pL);
  }
-
  return res;
 }

@@ -6230,12 +6236,10 @@ static int exprCompareVariable(
 ** just might result in some slightly slower code.  But returning
 ** an incorrect 0 or 1 could lead to a malfunction.
 **
-** If pParse is not NULL then TK_VARIABLE terms in pA with bindings in
-** pParse->pReprepare can be matched against literals in pB.  The
-** pParse->pVdbe->expmask bitmask is updated for each variable referenced.
-** If pParse is NULL (the normal case) then any TK_VARIABLE term in
-** Argument pParse should normally be NULL. If it is not NULL and pA or
-** pB causes a return value of 2.
+** If pParse is not NULL and SQLITE_EnableQPSG is off then TK_VARIABLE
+** terms in pA with bindings in pParse->pReprepare can be matched against
+** literals in pB.  The pParse->pVdbe->expmask bitmask is updated for
+** each variable referenced.
 */
 int sqlite3ExprCompare(
  const Parse *pParse,
@@ -6247,8 +6251,8 @@ int sqlite3ExprCompare(
  if( pA==0 || pB==0 ){
    return pB==pA ? 0 : 2;
  }
-  if( pParse && pA->op==TK_VARIABLE && exprCompareVariable(pParse, pA, pB) ){
-    return 0;
+  if( pParse && pA->op==TK_VARIABLE ){
+    return exprCompareVariable(pParse, pA, pB);
  }
  combinedFlags = pA->flags | pB->flags;
  if( combinedFlags & EP_IntValue ){
@@ -6443,18 +6447,66 @@ static int exprImpliesNotNull(
  return 0;
 }

+/*
+** Return true if the boolean value of the expression is always either
+** FALSE or NULL.
+*/
+static int sqlite3ExprIsNotTrue(Expr *pExpr){
+  int v;
+  if( pExpr->op==TK_NULL ) return 1;
+  if( pExpr->op==TK_TRUEFALSE && sqlite3ExprTruthValue(pExpr)==0 ) return 1;
+  v = 1;
+  if( sqlite3ExprIsInteger(pExpr, &v, 0) && v==0 ) return 1;
+  return 0;
+}
+
+/*
+** Return true if the expression is one of the following:
+**
+**    CASE WHEN x THEN y END
+**    CASE WHEN x THEN y ELSE NULL END
+**    CASE WHEN x THEN y ELSE false END
+**    iif(x,y)
+**    iif(x,y,NULL)
+**    iif(x,y,false)
+*/
+static int sqlite3ExprIsIIF(sqlite3 *db, const Expr *pExpr){
+  ExprList *pList;
+  if( pExpr->op==TK_FUNCTION ){
+    const char *z = pExpr->u.zToken;
+    FuncDef *pDef;
+    if( (z[0]!='i' && z[0]!='I') ) return 0;
+    if( pExpr->x.pList==0 ) return 0;
+    pDef = sqlite3FindFunction(db, z, pExpr->x.pList->nExpr, ENC(db), 0);
+    if( NEVER(pDef==0) ) return 0;
+    if( (pDef->funcFlags & SQLITE_FUNC_INLINE)==0 ) return 0;
+    if( NEVER(SQLITE_PTR_TO_INT(pDef->pUserData)!=INLINEFUNC_iif) ) return 0;
+  }else if( pExpr->op==TK_CASE ){
+    if( pExpr->pLeft!=0 ) return 0;
+  }else{
+    return 0;
+  }
+  pList = pExpr->x.pList;
+  assert( pList!=0 );
+  if( pList->nExpr==2 ) return 1;
+  if( pList->nExpr==3 && sqlite3ExprIsNotTrue(pList->a[2].pExpr) ) return 1;
+  return 0;
+}
+
 /*
 ** Return true if we can prove the pE2 will always be true if pE1 is
 ** true.  Return false if we cannot complete the proof or if pE2 might
 ** be false.  Examples:
 **
-**     pE1: x==5       pE2: x==5             Result: true
-**     pE1: x>0        pE2: x==5             Result: false
-**     pE1: x=21       pE2: x=21 OR y=43     Result: true
-**     pE1: x!=123     pE2: x IS NOT NULL    Result: true
-**     pE1: x!=?1      pE2: x IS NOT NULL    Result: true
-**     pE1: x IS NULL  pE2: x IS NOT NULL    Result: false
-**     pE1: x IS ?2    pE2: x IS NOT NULL    Result: false
+**     pE1: x==5        pE2: x==5             Result: true
+**     pE1: x>0         pE2: x==5             Result: false
+**     pE1: x=21        pE2: x=21 OR y=43     Result: true
+**     pE1: x!=123      pE2: x IS NOT NULL    Result: true
+**     pE1: x!=?1       pE2: x IS NOT NULL    Result: true
+**     pE1: x IS NULL   pE2: x IS NOT NULL    Result: false
+**     pE1: x IS ?2     pE2: x IS NOT NULL    Result: false
+**     pE1: iif(x,y)    pE2: x                Result: true
+**     PE1: iif(x,y,0)  pE2: x                Result: true
 **
 ** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has
 ** Expr.iTable<0 then assume a table number given by iTab.
@@ -6488,6 +6540,9 @@ int sqlite3ExprImpliesExpr(
  ){
    return 1;
  }
+  if( sqlite3ExprIsIIF(pParse->db, pE1) ){
+    return sqlite3ExprImpliesExpr(pParse,pE1->x.pList->a[0].pExpr,pE2,iTab);
+  }
  return 0;
 }

@@ -2814,7 +2814,10 @@ void sqlite3RegisterBuiltinFunctions(void){
 #endif /* SQLITE_ENABLE_MATH_FUNCTIONS */
    FUNCTION(sign,               1, 0, 0,      signFunc    ),
    INLINE_FUNC(coalesce,       -1, INLINEFUNC_coalesce, 0 ),
+    INLINE_FUNC(iif,             2, INLINEFUNC_iif,      0 ),
    INLINE_FUNC(iif,             3, INLINEFUNC_iif,      0 ),
+    INLINE_FUNC(if,              2, INLINEFUNC_iif,      0 ),
+    INLINE_FUNC(if,              3, INLINEFUNC_iif,      0 ),
  };
 #ifndef SQLITE_OMIT_ALTERTABLE
  sqlite3AlterFunctions();
@@ -4288,6 +4288,7 @@ int sqlite3_test_control(int op, ...){
        sqlite3ShowWinFunc(0);
 #endif
        sqlite3ShowSelect(0);
+        sqlite3ShowWhereTerm(0);
      }
 #endif
      break;
@@ -5053,7 +5053,7 @@ static int winOpen(

  int rc = SQLITE_OK;            /* Function Return Code */
 #if !defined(NDEBUG) || SQLITE_OS_WINCE
-  int eType = flags&0xFFFFFF00;  /* Type of file to open */
+  int eType = flags&0x0FFF00;  /* Type of file to open */
 #endif

  int isExclusive  = (flags & SQLITE_OPEN_EXCLUSIVE);
@@ -43,7 +43,7 @@
 %syntax_error {
  UNUSED_PARAMETER(yymajor);  /* Silence some compiler warnings */
  if( TOKEN.z[0] ){
-    sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
+    parserSyntaxError(pParse, &TOKEN);
  }else{
    sqlite3ErrorMsg(pParse, "incomplete input");
  }
@@ -111,6 +111,13 @@ struct TrigEvent { int a; IdList * b; };

 struct FrameBound     { int eType; Expr *pExpr; };

+/*
+** Generate a syntax error
+*/
+static void parserSyntaxError(Parse *pParse, Token *p){
+  sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p);
+}
+
 /*
 ** Disable lookaside memory allocation for objects that might be
 ** shared across database connections.
@@ -1156,7 +1163,7 @@ expr(A) ::= VARIABLE(X).     {
    Token t = X; /*A-overwrites-X*/
    assert( t.n>=2 );
    if( pParse->nested==0 ){
-      sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t);
+      parserSyntaxError(pParse, &t);
      A = 0;
    }else{
      A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);
@@ -938,6 +938,7 @@ void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExpr){
    pExpr = pExpr->pLeft;
  }
  if( pExpr==0 ) return;
+  if( ExprHasProperty(pExpr, EP_FromDDL) ) return;
  db->errByteOffset = pExpr->w.iOfst;
 }

@@ -6632,7 +6632,6 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){
  sqlite3_stmt *pStmt = 0;
  sqlite3_int64 nPage = 0;
  int pgSz = 0;
-  const char *zFilename;
  const char *zTail;
  char *zName = 0;
  int rc, i, j;
@@ -6660,17 +6659,15 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){
  if( nPage<1 ) goto dbtotxt_error;
  rc = sqlite3_prepare_v2(p->db, "PRAGMA databases", -1, &pStmt, 0);
  if( rc ) goto dbtotxt_error;
-  rc = 0;
  if( sqlite3_step(pStmt)!=SQLITE_ROW ){
-    zTail = zFilename = "unk.db";
+    zTail = "unk.db";
  }else{
-    zFilename = (const char*)sqlite3_column_text(pStmt, 2);
+    const char *zFilename = (const char*)sqlite3_column_text(pStmt, 2);
    if( zFilename==0 || zFilename[0]==0 ) zFilename = "unk.db";
    zTail = strrchr(zFilename, '/');
 #if defined(_WIN32)
    if( zTail==0 ) zTail = strrchr(zFilename, '\\');
 #endif
-    if( zTail ) zFilename = zTail;
  }
  zName = strdup(zTail);
  shell_check_oom(zName);
@@ -6681,7 +6678,6 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){
  rc = sqlite3_prepare_v2(p->db,
           "SELECT pgno, data FROM sqlite_dbpage ORDER BY pgno", -1, &pStmt, 0);
  if( rc ) goto dbtotxt_error;
-  rc = 0;
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    sqlite3_int64 pgno = sqlite3_column_int64(pStmt, 0);
    const u8 *aData = sqlite3_column_blob(pStmt, 1);
@@ -10411,7 +10407,10 @@ static int do_meta_command(char *zLine, ShellState *p){
  }else
 #endif /* !defined(SQLITE_SHELL_FIDDLE) */

-  if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){
+  if( c=='s' &&
+     (cli_strncmp(azArg[0], "scanstats",  n)==0 ||
+      cli_strncmp(azArg[0], "scanstatus", n)==0)
+  ){
    if( nArg==2 ){
      if( cli_strcmp(azArg[1], "vm")==0 ){
        p->scanstatsOn = 3;
@@ -11985,7 +11984,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
  char cWait = (char)qss; /* intentional narrowing loss */
  if( cWait==0 ){
  PlainScan:
-    assert( cWait==0 );
    while( (cin = *zLine++)!=0 ){
      if( IsSpace(cin) )
        continue;
@@ -12037,7 +12035,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
          if( *zLine != '/' )
            continue;
          ++zLine;
-          cWait = 0;
          CONTINUE_PROMPT_AWAITC(pst, 0);
          qss = QSS_SETV(qss, 0);
          goto PlainScan;
@@ -12049,7 +12046,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
          }
          deliberate_fall_through;
        case ']':
-          cWait = 0;
          CONTINUE_PROMPT_AWAITC(pst, 0);
          qss = QSS_SETV(qss, 0);
          goto PlainScan;
@@ -12237,7 +12233,10 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
 }

 static void echo_group_input(ShellState *p, const char *zDo){
-  if( ShellHasFlag(p, SHFLG_Echo) ) sqlite3_fprintf(p->out, "%s\n", zDo);
+  if( ShellHasFlag(p, SHFLG_Echo) ){
+    sqlite3_fprintf(p->out, "%s\n", zDo);
+    fflush(p->out);
+  }
 }

 #ifdef SQLITE_SHELL_FIDDLE
@@ -12698,6 +12697,15 @@ static void sayAbnormalExit(void){
  if( seenInterrupt ) eputz("Program interrupted.\n");
 }

+/* Routine to output from vfstrace
+*/
+static int vfstraceOut(const char *z, void *pArg){
+  ShellState *p = (ShellState*)pArg;
+  sqlite3_fputs(z, p->out);
+  fflush(p->out);
+  return 1;
+}
+
 #ifndef SQLITE_SHELL_IS_UTF8
 #  if (defined(_WIN32) || defined(WIN32)) \
   && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__)))
@@ -12934,8 +12942,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
         default: sqlite3_config(SQLITE_CONFIG_SERIALIZED);    break;
      }
    }else if( cli_strcmp(z,"-vfstrace")==0 ){
-      vfstrace_register("trace",0,(int(*)(const char*,void*))sqlite3_fputs,
-                        stderr,1);
      bEnableVfstrace = 1;
 #ifdef SQLITE_ENABLE_MULTIPLEX
    }else if( cli_strcmp(z,"-multiplex")==0 ){
@@ -13032,6 +13038,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
  }
  data.out = stdout;
+  if( bEnableVfstrace ){
+    vfstrace_register("trace",0,vfstraceOut, &data, 1);
+  }
 #ifndef SQLITE_SHELL_FIDDLE
  sqlite3_appendvfs_init(0,0,0);
 #endif
@@ -4034,12 +4034,15 @@ static const char *tclsh_main_loop(void){
 #ifdef WIN32
      "set new [list]\n"
      "foreach arg $argv {\n"
-        "if {[file exists $arg]} {\n"
+        "if {[string match -* $arg] || [file exists $arg]} {\n"
          "lappend new $arg\n"
        "} else {\n"
+          "set once 0\n"
          "foreach match [lsort [glob -nocomplain $arg]] {\n"
            "lappend new $match\n"
+            "set once 1\n"
          "}\n"
+          "if {!$once} {lappend new $arg}\n"
        "}\n"
      "}\n"
      "set argv $new\n"
@@ -187,7 +187,7 @@ static int SQLITE_TCLAPI hexio_write(
 }

 /*
-** USAGE:   hexio_get_int   HEXDATA
+** USAGE:   hexio_get_int [-littleendian] HEXDATA
 **
 ** Interpret the HEXDATA argument as a big-endian integer.  Return
 ** the value of that integer.  HEXDATA can contain between 2 and 8
@@ -205,12 +205,20 @@ static int SQLITE_TCLAPI hexio_get_int(
  const unsigned char *zIn;
  unsigned char *aOut;
  unsigned char aNum[4];
+  int bLittle = 0;

-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA");
+  if( objc==3 ){
+    Tcl_Size n;
+    char *z = Tcl_GetStringFromObj(objv[1], &n);
+    if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){
+      bLittle = 1;
+    }
+  }
+  if( (objc-bLittle)!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA");
    return TCL_ERROR;
  }
-  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
+  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn);
  aOut = sqlite3_malloc64( 1 + nIn/2 );
  if( aOut==0 ){
    return TCL_ERROR;
@@ -223,7 +231,11 @@ static int SQLITE_TCLAPI hexio_get_int(
    memcpy(&aNum[4-nOut], aOut, nOut);
  }
  sqlite3_free(aOut);
-  val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  if( bLittle ){
+    val = (int)((u32)aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
+  }else{
+    val = (int)((u32)aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  }
  Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
  return TCL_OK;
 }
@@ -1302,6 +1302,10 @@ void sqlite3TreeViewTrigger(
 ** accessible to the debugging, and to avoid warnings about unused
 ** functions.  But these routines only exist in debugging builds, so they
 ** do not contaminate the interface.
+**
+** See Also:
+**
+**     sqlite3ShowWhereTerm() in where.c
 */
 void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); }
 void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);}
@@ -2332,7 +2332,7 @@ static int whereInScanEst(
 #endif /* SQLITE_ENABLE_STAT4 */


-#ifdef WHERETRACE_ENABLED
+#if defined(WHERETRACE_ENABLED) || defined(SQLITE_DEBUG)
 /*
 ** Print the content of a WhereTerm object
 */
@@ -2376,6 +2376,9 @@ void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
    sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
  }
 }
+void sqlite3ShowWhereTerm(WhereTerm *pTerm){
+  sqlite3WhereTermPrint(pTerm, 0);
+}
 #endif

 #ifdef WHERETRACE_ENABLED
@@ -3561,7 +3564,6 @@ static int whereUsablePartialIndex(
    if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
    pWhere = pWhere->pRight;
  }
-  if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0;
  for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
    Expr *pExpr;
    pExpr = pTerm->pExpr;
@@ -615,6 +615,7 @@ static Expr *removeUnindexableInClauseTerms(
        pNew->pLeft->x.pList = pLhs;
      }
      pSelect->pEList = pRhs;
+      pSelect->selId = ++pParse->nSelect; /* Req'd for SubrtnSig validity */
      if( pLhs && pLhs->nExpr==1 ){
        /* Take care here not to generate a TK_VECTOR containing only a
        ** single value. Since the parser never creates such a vector, some
@@ -141,15 +141,6 @@ do_test 2.0 {
 | end c-b92b.txt.db
 }]} {}

-# This test only works with the legacy RC4 PRNG
-if 0 {
-  prng_seed 0 db
-  do_catchsql_test 2.1 {
-  SELECT count(*) FROM sqlite_schema;
-  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
-  INSERT INTO t1(a) SELECT randomblob(null) FROM c;
-  } {1 {database disk image is malformed}}
-}

 reset_db
 if {![info exists ::G(perm:presql)]} {
@@ -1232,12 +1232,18 @@ db nullvalue {}
 # EVIDENCE-OF: R-13943-13592 A NULL result is considered untrue when
 # evaluating WHEN terms.
 #
-do_execsql_test e_expr-21.4.1 {
+do_execsql_test e_expr-21.4.1a {
  SELECT CASE WHEN NULL THEN 'A' WHEN 1 THEN 'B' END, iif(NULL,8,99);
 } {B 99}
-do_execsql_test e_expr-21.4.2 {
+do_execsql_test e_expr-21.4.1b {
+  SELECT CASE WHEN NULL THEN 'A' WHEN 1 THEN 'B' END, if(NULL,8,99);
+} {B 99}
+do_execsql_test e_expr-21.4.2a {
  SELECT CASE WHEN 0 THEN 'A' WHEN NULL THEN 'B' ELSE 'C' END, iif(0,8,99);
 } {C 99}
+do_execsql_test e_expr-21.4.2b {
+  SELECT CASE WHEN 0 THEN 'A' WHEN NULL THEN 'B' ELSE 'C' END, if(0,8,99);
+} {C 99}

 # EVIDENCE-OF: R-38620-19499 In a CASE with a base expression, the base
 # expression is evaluated just once and the result is compared against
@@ -1969,9 +1975,12 @@ do_execsql_test e_expr-37.5 {
 # EVIDENCE-OF: R-55532-10108 Values 1, 1.0, 0.1, -0.1 and '1english' are
 # considered to be true.
 #
-do_execsql_test e_expr-37.6 {
+do_execsql_test e_expr-37.6a {
   SELECT CASE WHEN 1 THEN 'true' ELSE 'false' END, iif(1,'true','false');
 } {true true}
+do_execsql_test e_expr-37.6b {
+   SELECT CASE WHEN 1 THEN 'true' ELSE 'false' END, if(1,'true');
+} {true true}
 do_execsql_test e_expr-37.7 {
   SELECT CASE WHEN 1.0 THEN 'true' ELSE 'false' END, iif(1.0,'true','false');
 } {true true}
@@ -0,0 +1,31 @@
+# 2024-11-20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for sqlite3_error_offset()
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test errofst1-1.1 {
+  CREATE TABLE t1 as select 1 as aa;
+  CREATE VIEW t2 AS
+     WITH t3 AS (SELECT 1 FROM t1 AS bb, t1 AS cc WHERE cc.aa <= sts.aa)
+     SELECT 1 FROM t3 AS dd;
+}
+do_catchsql_test errofst1-1.2 {
+  SELECT * FROM t2;
+} {1 {no such column: sts.aa}}
+do_test errofst1-1.3 {
+  sqlite3_error_offset db
+} {-1}
+
+finish_test
@@ -216,6 +216,14 @@ do_faultsim_test 8.4 -prep {
 } -test {
  faultsim_test_result {0 3}
 }
+do_faultsim_test 8.5 -prep { 
+  faultsim_restore_and_reopen
+  db func mit mit
+} -body {
+  execsql { SELECT mit(matchinfo(t8, 'l')) FROM t8 WHERE t8 MATCH '"a b c"' }
+} -test {
+  faultsim_test_result {0 3}
+}

 do_test 9.0 {
  faultsim_delete_and_reopen
@@ -192,6 +192,31 @@ do_execsql_test 3.4 {
  1 2 3 4 5 6
 }

-
+# 2024-11-20 https://sqlite.org/forum/forumpost/0b9ded2f8428ac00
+#
+# Bug in SubrtnSig logic.  If a SELECT statement is copied and the copy
+# is subsequently modified, we need to change the Select.selId on the
+# copy so that when the copy is used to generate code, the SubrtnSig
+# logic won't try to substitute the original SELECT in place of the
+# copy which is now different.
+#
+do_execsql_test 3.5 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1 (a int UNIQUE);
+  CREATE TABLE t2 (b int UNIQUE);
+  INSERT INTO t1 VALUES (1);
+  INSERT INTO t2 VALUES (1), (2);
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) = (1, 1);
+} {1 1}
+do_execsql_test 3.6 {
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) IN ((1, 1));
+} {1 1}
+do_execsql_test 3.7 {
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) = (1, 2);
+} {1 2}
+do_execsql_test 3.8 {
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) IN ((1, 2));
+} {1 2}

 finish_test
@@ -745,6 +745,30 @@ do_execsql_test json102-1610 {
  5 {{"b":9}} json {{"b":9}} text    {{"b":9}} json       \
  6 {}        null {}        null    {}        null
 ]
+do_execsql_test json102-1620 {
+  DELETE FROM t1;
+  INSERT INTO t1(x) VALUES('[null,123,4.5,"six",[7,8],{"b":9}]');
+  WITH c(y) AS (VALUES(0),(1),(2),(3),(4),(5),(6))
+  SELECT
+    y,
+    x->y AS '->',
+    CASE WHEN subtype(if(json_valid(x),x->y)) THEN 'json'
+         ELSE typeof(x->y) END AS 'type',
+    x->>y AS '->>',
+    CASE WHEN subtype(x->>y) THEN 'json' ELSE typeof(x->>y) END AS 'type',
+    json_extract(x,format('$[%d]',y)) AS 'json_extract',
+    CASE WHEN subtype(json_extract(x,format('$[%d]',y)))
+      THEN 'json' ELSE typeof(json_extract(x,format('$[%d]',y))) END AS 'type'
+  FROM c, t1 ORDER BY y;    
+} [list \
+  0 null      json {}        null    {}        null       \
+  1 123       json 123       integer 123       integer    \
+  2 4.5       json 4.5       real    4.5       real       \
+  3 {"six"}   json six       text    six       text       \
+  4 {[7,8]}   json {[7,8]}   text    {[7,8]}   json       \
+  5 {{"b":9}} json {{"b":9}} text    {{"b":9}} json       \
+  6 {}        null {}        null    {}        null
+]

 reset_db
 do_execsql_test json102-1700 {
@@ -15,6 +15,10 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl

 proc sql {zSql} {
+  if {0 && $::debug_op} {
+    puts stderr "$zSql ;"
+    flush stderr
+  }
  uplevel db eval [list $zSql]
  #puts stderr "$zSql ;"
 }
@@ -67,11 +71,13 @@ proc x_to_y {x} {
 #   delete_rows XVALUES
 #
 proc savepoint {zName} {
+  if {$::debug_op} { puts stderr "savepoint $zName" ; flush stderr }
  catch { sql "SAVEPOINT $zName" }
  lappend ::lSavepoint [list $zName [array get ::aEntry]]
 }

 proc rollback {zName} {
+  if {$::debug_op} { puts stderr "rollback $zName" ; flush stderr }
  catch { sql "ROLLBACK TO $zName" }
  for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
    set zSavepoint [lindex $::lSavepoint $i 0]
@@ -89,6 +95,7 @@ proc rollback {zName} {
 }

 proc release {zName} {
+  if {$::debug_op} { puts stderr "release $zName" ; flush stderr }
  catch { sql "RELEASE $zName" }
  for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
    set zSavepoint [lindex $::lSavepoint $i 0]
@@ -104,6 +111,7 @@ proc release {zName} {
 }

 proc insert_rows {lX} {
+  if {$::debug_op} { puts stderr "insert_rows $lX" ; flush stderr }
  foreach x $lX {
    set y [x_to_y $x]

@@ -116,6 +124,7 @@ proc insert_rows {lX} {
 }

 proc delete_rows {lX} {
+  if {$::debug_op} { puts stderr "delete_rows $lX" ; flush stderr }
  foreach x $lX {
    # Update database [db]
    sql "DELETE FROM t1 WHERE x = $x"
@@ -164,6 +173,11 @@ proc random_integers {nRes nRange} {
 } 
 #-------------------------------------------------------------------------

+set ::debug_op 0
+proc debug_ops {} {
+  set ::debug_op 1
+}
+
 proc database_op {} {
  set i [expr int(rand()*2)] 
  if {$i==0} {
@@ -185,9 +199,6 @@ proc savepoint_op {} {
  set C [lindex $cmds [expr int(rand()*6)]]
  set N [lindex $names [expr int(rand()*5)]]

-  #puts stderr "   $C $N ;  "
-  #flush stderr
-
  $C $N
  return ok
 }
@@ -73,5 +73,15 @@ do_execsql_test subtype1-320 {
   WHERE json_quote(y)='"1"';
 } {1 {"1"}}

+# 2024-11-26
+# subtype survives if()
+#
+do_execsql_test subtype1-400 {
+  WITH t400(id,j) AS (VALUES
+    (1,'{a:{x:1,y:2},b:{x:3,y:4}}'),
+    (2,'not json')
+  )
+  SELECT id, subtype(if(json_valid(j,6),j->'a')) FROM t400;
+} {1 74 2 0}

 finish_test
@@ -57,6 +57,7 @@ do_execsql_test 1.0 {
  INSERT INTO t1 VALUES(1,9999);
 }

+forcedelete link.db
 do_test 2.0 {
  createWin32Symlink link.db test.db
 } {}
@@ -87,12 +88,13 @@ do_test 3.4 {
  db3 close
 } {}

+# The -nofollow option does not work on Windows
 do_test 3.5 {
  list [catch {
    sqlite3 db4 link.db -nofollow true
    execsql { SELECT x, y FROM t1; } db4
  } res] $res
-} {1 {unable to open database file}}
+} {0 {1 9999}}

 catch {db4 close}

@@ -465,6 +465,7 @@ if {[string compare -nocase script [lindex $argv 0]]==0} {
 # number of milliseconds in the argument.
 #
 proc elapsetime {ms} {
+  if {$ms==""} {set ms 0}
  set s [expr {int(($ms+500.0)*0.001)}]
  set hr [expr {$s/3600}]
  set mn [expr {($s/60)%60}]
@@ -1603,6 +1604,8 @@ proc run_testset {} {
     SELECT pltfm, count(*) FROM jobs WHERE pltfm IS NOT NULL
      ORDER BY 2 DESC LIMIT 1
  } break
+  if {$totalerr==""} {set totalerr 0}
+  if {$totaltest==""} {set totaltest 0}
  puts "$totalerr errors out of $totaltest tests in $et $pltfm"
  trdb eval {
     SELECT DISTINCT substr(svers,1,79) as v1 FROM jobs WHERE svers IS NOT NULL
@@ -35,43 +35,6 @@ proc cond_incr_sync_count {adj} {
  }
 }

-proc set_tvfs_hdr {file args} {
-
-  # Set $nHdr to the number of bytes in the wal-index header:
-  set nHdr 48
-  set nInt [expr {$nHdr/4}]
-
-  if {[llength $args]>2} {
-    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
-  }
-
-  set blob [tvfs shm $file]
-  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
-
-  if {[llength $args]} {
-    set ia [lindex $args 0]
-    set ib $ia
-    if {[llength $args]==2} {
-      set ib [lindex $args 1]
-    }
-    binary scan $blob a[expr $nHdr*2]a* dummy tail
-    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
-    tvfs shm $file $blob
-  }
-
-  binary scan $blob ${fmt}${nInt} ints
-  return $ints
-}
-
-proc incr_tvfs_hdr {file idx incrval} {
-  set ints [set_tvfs_hdr $file]
-  set v [lindex $ints $idx]
-  incr v $incrval
-  lset ints $idx $v
-  set_tvfs_hdr $file $ints
-}
-
-
 #-------------------------------------------------------------------------
 # Test case wal2-1.*:
 #
@@ -87,3 +87,43 @@ proc wal_fix_walindex_cksum {hdrvar} {
  lset hdr 10 $c1
  lset hdr 11 $c2
 }
+
+# This command assumes that $file is the name of a database file opened
+# in wal mode using a [testvfs] VFS. It returns a list of the 12 32-bit
+# integers that make up the wal-index-header for the named file.
+#
+proc set_tvfs_hdr {file args} {
+
+  # Set $nHdr to the number of bytes in the wal-index header:
+  set nHdr 48
+  set nInt [expr {$nHdr/4}]
+
+  if {[llength $args]>2} {
+    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
+  }
+
+  set blob [tvfs shm $file]
+  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
+
+  if {[llength $args]} {
+    set ia [lindex $args 0]
+    set ib $ia
+    if {[llength $args]==2} {
+      set ib [lindex $args 1]
+    }
+    binary scan $blob a[expr $nHdr*2]a* dummy tail
+    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
+    tvfs shm $file $blob
+  }
+
+  binary scan $blob ${fmt}${nInt} ints
+  return $ints
+}
+
+proc incr_tvfs_hdr {file idx incrval} {
+  set ints [set_tvfs_hdr $file]
+  set v [lindex $ints $idx]
+  incr v $incrval
+  lset ints $idx $v
+  set_tvfs_hdr $file $ints
+}
@@ -15,6 +15,7 @@ Options:
   --info               Show info on existing SQLite TCL extension installs
   --install-only       Install an extension previously build
   --uninstall          Uninstall the extension
+   --destdir DIR        Installation root (used by "make install DESTDIR=...")

 Other options are retained and passed through into the compiler.}

@@ -25,6 +26,7 @@ set uninstall 0
 set infoonly 0
 set CC {}
 set OPTS {}
+set DESTDIR ""; # --destdir "$(DESTDIR)"
 for {set ii 0} {$ii<[llength $argv]} {incr ii} {
  set a0 [lindex $argv $ii]
  if {$a0=="--install-only"} {
@@ -42,6 +44,9 @@ for {set ii 0} {$ii<[llength $argv]} {incr ii} {
  } elseif {$a0=="--cc" && $ii+1<[llength $argv]} {
    incr ii
    set CC [lindex $argv $ii]
+  } elseif {$a0=="--destdir" && $ii+1<[llength $argv]} {
+    incr ii
+    set DESTDIR [lindex $argv $ii]
  } elseif {[string match -* $a0]} {
    append OPTS " $a0"
  } else {
@@ -196,7 +201,26 @@ if {$install} {
  #
  set DEST {}
  foreach dir $auto_path {
-    if {[file writable $dir]} {
+    if {[string match //*:* $dir]} {
+      # We can't install to //zipfs: paths
+      continue
+    } elseif {"" ne $DESTDIR && ![file writable $DESTDIR]} {
+      # In the common case, ${DESTDIR}${dir} will not exist when we
+      # get to this point of the installation, and the "is writable?"
+      # check just below this will fail for that case.
+      #
+      # Assumption made for simplification's sake: if ${DESTDIR} is
+      # not writable, no part of the remaining path will
+      # be. ${DESTDIR} is typically used by OS package maintainers,
+      # not normal installations, and it "shouldn't" ever happen that
+      # the DESTDIR is read-only while the target ${DESTDIR}${prefix}
+      # is not, as it's typical for such installations to create
+      # ${prefix} on-demand under ${DESTDIR}.
+      break
+    }
+    set dir ${DESTDIR}$dir
+    if {[file writable $dir] || "" ne $DESTDIR} {
+      # the dir will be created later ^^^^^^^^
      set DEST $dir
      break
    } elseif {[glob -nocomplain $dir/sqlite3*/pkgIndex.tcl]!=""} {
@@ -214,7 +238,7 @@ if {$install} {
    puts "to work around this problem.\n"
    puts "These are the (unwritable) \$auto_path directories:\n"
    foreach dir $auto_path {
-      puts "  *  $dir"
+      puts "  *  ${DESTDIR}$dir"
    }
    exit 1
  }
@@ -6,7 +6,7 @@
 #
 # Usage example:
 #
-#     tclsh mkccode.tcl demoapp.c.in >demoapp.c
+#     tclsh mkccode.tcl -DENABLE_FEATURE_XYZ demoapp.c.in >demoapp.c
 #
 # The demoapp.c.in file contains a mixture of C code, TCL script, and
 # processing directives used by mktclsqliteprog.tcl to build the final C-code
@@ -33,29 +33,63 @@
 #      then all of the text in the input file is converted into C-language
 #      string literals.
 #
+# IFDEF macro
+# IFNDEF macro
+# ELSE
+# ENDIF
+#
+#      The text from "IFDEF macro" down to the next ELSE or ENDIF is
+#      included only if -Dmacro appears as a command-line argument.
+#      The "IFNDEF macro" simply inverts the initial test.
+#
 # None of the control directives described above will nest.  Only the
 # top-level input file ("demoapp.c.in" in the example) is interpreted.
 # referenced files are copied verbatim.
 #
-if {[llength $argv]!=1} {
-  puts stderr "Usage: $argv0 TEMPLATE >OUTPUT"
+proc usage {} {
+  puts stderr "Usage: $::argv0 \[OPTIONS\] TEMPLATE >OUTPUT"
  exit 1
 }
-set infile [lindex $argv 0]
+set infile {}
+foreach ax $argv {
+  if {[string match -D* $ax]} {
+    if {[string match *=* $ax]} {
+      regexp -- {-D([^=]+)=(.*)} $ax all name value
+      set DEF($name) $value
+    } else {
+      set DEF([string range $ax 2 end]) 1
+    }
+    continue
+  }
+  if {[string match -* $ax]} {
+    puts stderr "$::argv0:  Unknown option \"$ax\""
+    usage
+  }
+  if {$infile!=""} {
+    puts stderr "$::argv0:  Surplus argument: \"$ax\""
+    usage
+  }
+  set infile $ax
+}
 set ROOT [file normalize [file dir $argv0]/..]
 set HOME [file normalize [file dir $infile]]
 set in [open $infile rb]
 puts [subst {/* DO NOT EDIT
 **
-** This file was generated by \"$argv0 $infile\".
+** This file was generated by \"$argv0 $argv\".
 ** To make changes, edit $infile then rerun the generator
 ** command.
 */}]
 set instr 0
+set omit {}
+set nomit 0
+set ln 0
 while {1} {
  set line [gets $in]
+  incr ln
  if {[eof $in]} break
  if {[regexp {^INCLUDE (.*)} $line all path]} {
+    if {$nomit>0 && [string match *1* $omit]} continue
    if {0} {
      # https://github.com/msteveb/jimtcl/issues/320
      regsub {^\$ROOT\y} $path $ROOT path
@@ -91,10 +125,43 @@ while {1} {
    puts "/* END_STRING */"
    continue
  }
-  if {$instr} {
+  if {[regexp {^IFNDEF +([A-Za-z_0-9]+)} $line all name]} {
+    set omit $omit[info exists DEF($name)]
+    incr nomit
+    continue
+  }
+  if {[regexp {^IFDEF +([A-Za-z_0-9]+)} $line all name]} {
+    set omit $omit[expr {![info exists DEF($name)]}]
+    incr nomit
+    continue
+  }
+  if {[regexp {^ELSE} $line]} {
+    if {!$nomit} {
+      puts stderr "$infile:$ln: ELSE without a prior IFDEF"
+      exit 1
+    }
+    set omit [string range $omit 0 end-1][expr {![string index $omit end]}]
+    continue
+  }
+  if {[regexp {^ENDIF} $line]} {
+    if {!$nomit} {
+      puts stderr "$infile:$ln: ENDIF without a prior IFDEF"
+      exit 1
+    }
+    incr nomit -1
+    set omit [string range $omit 0 [expr {$nomit-1}]]
+    continue
+  }
+  if {$nomit>0 && [string match *1* $omit]} {
+    # noop
+  } elseif {$instr} {
    set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line]
    puts "\"$x\\n\""
  } else {
    puts $line
  }
 }
+if {$nomit} {
+  puts stderr "$infile:$ln: One or more unterminated IFDEFs"
+  exit 1
+}
@@ -3,6 +3,8 @@
 ** text on standard output.
 */
 #define TCLSH_INIT_PROC sqlite3_analyzer_init_proc
+IFDEF INCLUDE_SQLITE3_C
+#undef SQLITE_ENABLE_DBSTAT_VTAB
 #define SQLITE_ENABLE_DBSTAT_VTAB 1
 #undef SQLITE_THREADSAFE
 #define SQLITE_THREADSAFE 0
@@ -14,9 +16,10 @@
 #define SQLITE_DEFAULT_MEMSTATUS 0
 #define SQLITE_MAX_EXPR_DEPTH 0
 #define SQLITE_OMIT_LOAD_EXTENSION 1
-#if !defined(SQLITE_AMALGAMATION) && !defined(USE_EXTERNAL_SQLITE)
 INCLUDE sqlite3.c
-#endif
+ELSE
+#include "sqlite3.h"
+ENDIF
 INCLUDE $ROOT/src/tclsqlite.c

 #if defined(_WIN32)
Author	SHA1	Message	Date
drh	dc7b94e63f	Simplification to the fix in the prior check-in. FossilOrigin-Name: f79cb748c9447ca4d2184a24219feb19571eb79870e0bedf14704743a3d5f8b9	2024-11-28 02:09:27 +00:00
drh	3ff6aa6e39	The sqlite3ExprCompare() routine should always compare the same variables as equal to one another, regardless of whether or not QPSG is set. FossilOrigin-Name: df95d5209a77c0302e08ea7b1f248646fb2a062ba5beca898678aee0a1d03a3a	2024-11-28 01:47:44 +00:00
drh	bfb7f77a8c	Add two NEVER() conditions. FossilOrigin-Name: 4f358ad20dc8a45e7aaa1a1996b40abd39caf34be6556ed54e1e0c80572b7ac0	2024-11-28 00:58:37 +00:00
drh	60ad526639	Enhancements to sqlite3ExprImpliesExpr() so that it realizes that "iif(A,B)" implies "A". FossilOrigin-Name: d96ecbee59b50eca25447456dfff2c168e8eade7a68e542932d5829f97debb31	2024-11-28 00:34:20 +00:00
dan	0bce1d6c9f	Fix an inaccuracy in the documentation in fts5.h. FossilOrigin-Name: ec3ca6f17972a9624018ae352e5a4be8dc34602a4569c80e827e09cd7fbed7f7	2024-11-27 18:01:25 +00:00
drh	cf04895832	Allow the iif() function to work with two arguments. The third argument is then assumed to be NULL. Also allow if() as an alternative spelling for iif(). FossilOrigin-Name: 1e405c39a8405e2081ce061559d1aacc70055fc74ab86d886b973076a3fb124b	2024-11-26 23:40:54 +00:00
drh	966a158831	New test cases. FossilOrigin-Name: f1cb1a815a09d71eb52d5c7f2f4e5c7689535232dee365e46ac41cce7fd86c01	2024-11-26 23:30:49 +00:00
drh	f12e5d1a0f	Provide a two-argument version of the iif() function, plus an alternative spelling that only requires a single "i". FossilOrigin-Name: a251ee645e11e24b67473d8a5bd3f8b72fde1ac9d5fda074f5da2297deb2faa8	2024-11-26 20:56:03 +00:00
dan	3e4ccc3208	Update fts5 so that the xInstToken() extension API works with prefix queries. FossilOrigin-Name: 41b79bde4a4b6510b41cc9175ff2527fbd3ca9209812361783754f24cf67ef9e	2024-11-26 20:46:18 +00:00
dan	e2a3c7232b	Update documentation in fts5.h. FossilOrigin-Name: f1e44f703acb415e2ff7d7f87fa05fc874ef1c432095bb3ccb45c6ad2fd085a9	2024-11-26 20:16:34 +00:00
stephan	19a6315ccf	Patch autosetup to rename --debug to --autosetup-debug so we can use --debug/--enable-debug for its historical purpose. Update autosetup/README.md with details of the patch. FossilOrigin-Name: 00dffd10f0c77fed53bdbf1536f6ef9b6076775f1b9a10794b87e3ace33ca1d9	2024-11-26 19:19:12 +00:00
stephan	58cb5644bc	Patch autosetup to rename --debug to --autosetup-debug so we can use --debug/--enable-debug for its historical purpose. --with-debug is now an alias for --enable-debug but can be removed entirely if all scripts which have been adjusted for the autosetup build are edited to (re)use the older flag name. FossilOrigin-Name: 3296c8d30559eafa20f4748339177149172ecf79b019c42ed0d5ee9cf62a666a	2024-11-26 19:02:16 +00:00
stephan	04a556acf2	Move some of the in-comment configure script help text into the --help text. No functional changes. FossilOrigin-Name: ee8449c482b6f9cfb82fa2417da194fc9aa56f54ef3ac83d368537181634ab1c	2024-11-26 16:12:05 +00:00
drh	0dd542f79e	Minor tweak to ./configure --help output. FossilOrigin-Name: 7e634d0569667ce1e17f4a611039cbb726078dc2e6be1f1b19655dc569170ce5	2024-11-26 16:04:02 +00:00
stephan	f2d56bc578	Replace some outdated docs re. soname with a link to the soname forum post. FossilOrigin-Name: 9225701cb88cb4629a12b45de5743378448b1d2ddb351916b77cfde638698070	2024-11-26 03:15:34 +00:00
stephan	b56f179e45	Correct duplicated soname linker flags, one of them mis-named. FossilOrigin-Name: 87dfd92d3a5a6a09c32c70e724a3f341bc63b9c6dfa0aa94fbbbdd45cc9c29f3	2024-11-25 17:07:58 +00:00
drh	8ff67df7ac	Fix another issue in argument expansion on Windows for tclsqlite3.c in interpreter mode. Problem introduced by check-in [9b87ea219bce5689] and unfixed by [cd942dce148c9d8f]. FossilOrigin-Name: 0fe1622cec95b7ebecc127ee57a08113d3da1dadbe72c03a13d6751b3043e50f	2024-11-22 17:41:00 +00:00
stephan	7398e279a7	Remove a flaky JS test which has a result depending on unrepredictable context. That same feature is more reliably tested at a later point in the same script. FossilOrigin-Name: 3d6ae13805bdba4c73b7443f20073264cdd157299cb911228600e1528a136bb1	2024-11-22 16:45:43 +00:00
stephan	de7e2b178d	In order to support package builds, like OpenBSD's, which set a custom soname on libsqlite3.so, extend the --soname configure flag to allow arbitrary soname values. FossilOrigin-Name: 9c1c1b99837efe3704778c594a65e0cd4cd0a8ec492044d3bb07f1b37a692c6a	2024-11-22 16:24:55 +00:00
stephan	5c945264b0	Move handling of the tempstore feature flag into the OPT_FEATURE_FLAGS list and remove the CFLAGS.libsqlite3 makefile var which exists solely to account for the tempstore being tracked separately from the other feature flags. FossilOrigin-Name: 2df5065d118ca4abcb4f285d07819e73c5e32f7e470d30eabc7e3d999ec8d0f2	2024-11-22 14:18:23 +00:00
stephan	dcaf03fccf	Makefile doc cleanups. No functional changes. FossilOrigin-Name: 64add0ac706101c53e2d2877fdc0d1ccd071814cae969768d2741cee05f23c01	2024-11-22 14:12:18 +00:00
stephan	7a13effa20	Remove the libsqlite3.so.3 link from the installation process, as it now serves no functional purpose. We retain libsqlite3.so.0 (A) for compatibility with clients linked against legacy builds and (B) 0 is still valid as the library's ABI version. FossilOrigin-Name: 91bd9813e04e34e088d5e73bfe2a4f338513363d9a99095a232b811c6b616354	2024-11-22 13:47:12 +00:00
stephan	0cf7131555	Add links to [/forumpost/5a3b44f510df8ded\|the SONAME discussion] at relevant places in auto.def and the makefiles. No functional changes. FossilOrigin-Name: c78dcc318a36af123676e54d85fc7f980c21b0ea366d164a96e2d998286d3813	2024-11-22 13:22:25 +00:00
drh	ed271dc7ea	Fix harmless scanbuild warnings caused by the introduction of the ".dbtotxt" command into the CLI by check-in [b43acf5a8cd4a5ef]. FossilOrigin-Name: 554d8fbd865436ace900859874b6c8c7e1b782184158a86b7788644e27bd1997	2024-11-22 12:29:35 +00:00
drh	38fdb2a857	Fix some harmless scanbuild warnings in the shell. FossilOrigin-Name: 9ba1c9b505d459366274043c1c5327f1a9d4e15d3564d99f8a15926af1d6d247	2024-11-22 12:07:21 +00:00
drh	f0a9af433a	Fix harmless compiler warning caused by [c77a4a42f2e3d164]. FossilOrigin-Name: 063690d2c5cc2a887055b9141d3fcdb4fd2fa35d069c9a39b81903c531c73fcf	2024-11-22 12:00:31 +00:00
drh	9edb535bdd	Make ".scanstatus" an undocumented alternative name for ".scanstats" in the CLI. FossilOrigin-Name: f20688efc2bc54648618b0aa2593a771f455ee8cc703b52273452d15e680b67c	2024-11-21 20:57:11 +00:00
stephan	a8c68e551d	Fix a bug-in-waiting (one too few list elements) in [b6bd25bd769e]. FossilOrigin-Name: e527dcd016fbeb55b0e613a0b3b95350bbc8b3ee601bbadfedaf6bd5458b3570	2024-11-21 20:10:38 +00:00
stephan	d204a83c9b	Add the --scanstatus configure flag to set -DSQLITE_ENABLE_STMT_SCANSTATUS. FossilOrigin-Name: b6bd25bd769e4286ad0b8ca5059bc9da52188cefe92525d8994aa921768aa750	2024-11-21 20:07:01 +00:00
stephan	2f70d5838a	Fix a JS test which was broken when OMIT_PROGRESS_CALLBACK was _not_ used. FossilOrigin-Name: fba23150b5b57980f8ed2656a2325f95215ca1e13fc3449b5eec804831dd65c8	2024-11-21 15:53:31 +00:00
drh	9bf5bea607	Document that -nofollow does not work on Windows. Fix Windows symlink test cases for when the test suite is run as administrator. FossilOrigin-Name: 4de8a75ec5a2e3655a2d0c5a9333b58f8b48b681255f0a01866b610ae6249bc2	2024-11-21 01:50:01 +00:00
dan	032bcf72fb	Fix compiler warnings caused by variable shadowing. FossilOrigin-Name: 211b305791980b24c4192ffc57a0471473de3fca32bfc146c0eeacedef7a88aa	2024-11-20 20:39:18 +00:00
drh	d02833bf05	New configuration option --dynlink-tools causes some command-line tools like sqldiff and sqlite3_analyzer to link against the libsqlite3.so system library rather than being built-in. Caution: sqlite3_analyzer requires the SQLITE_ENABLE_DBSTAT_VTAB compile-time option on its SQLite library in order to work, so do not use --dynlink-tools to build sqlite3_analyzer without it. FossilOrigin-Name: 314c606dd36e03d2ded899c536585ea21250af56b553fa4c96dc714cb5099522	2024-11-20 16:21:34 +00:00
drh	914e32f24a	Use shared-library flags, not shared-object flags, to create shared libraries. This makes no difference on Linux, but is required for Mac. FossilOrigin-Name: bfaa3ee7588b6f6be9799882c6ed6472357d988fac31512d5e92fe28f57029c6	2024-11-20 16:17:01 +00:00
drh	2722e2e822	Bug fix in the SubrtnSig logic from [c9a3498113074bbc], if a subquery is copied and then changes are made to the copy, be sure to give the copy a unique Select.selId value so that the original will not be substituted in place of the modified copy. [forum:/forumpost/0b9ded2f8428ac00\|Forum post 0b9ded2f8428ac00]. FossilOrigin-Name: 19d1bede5654bcfa9f7a151b9b2616a3d10873b8e3f8cf54a3a7e810da08f844	2024-11-20 14:59:32 +00:00
drh	d4a65cfc51	Provide the sqlite3ShowWhereTerm() interface callable interactively from a debugger, when compiling with SQLITE_DEBUG. FossilOrigin-Name: c77a4a42f2e3d16427a69295436efe36335e56e97abd0efba814092498a8dea8	2024-11-20 14:19:44 +00:00
drh	9566982328	Do not report an sqlite3_error_offset() for errors that occur inside of views or triggers, since the text of those elements is not part of the original query. FossilOrigin-Name: bf66c6dfc25c2562a4e6a5b24dd1660213a8fefbb5763e7583b87fb06dbaaf43	2024-11-20 11:34:16 +00:00
stephan	a34fbd1b9b	Add fix from [bfab759611b0] to sqltclsh. Remove some now-unneeded has_tclsh84/5 checks in cases where jimsh can be used. FossilOrigin-Name: 9d389b998317c5006876f2f40d56cd763d1555bbeaf48b2bca9bd46dcd02b80e	2024-11-19 21:19:12 +00:00
drh	18c9e2638e	Add missing $TCL_LIBS to the link for sqlite3_analyzer. FossilOrigin-Name: bfab759611b0562837d8733ce56591854db08c122956524799dc1d5b3e2d0279	2024-11-19 21:14:48 +00:00
stephan	457d9384a2	Rename --link-tools-dynamically to --dynlink-tools, per discussion. FossilOrigin-Name: 50b9f6fde44f9afda27e47badde6115ab6d9ccd5e22b446a8d8127499848815f	2024-11-19 20:21:40 +00:00
stephan	b81e11b25b	Move the in-comment-code commentary about --with-debug, from [7b14309be4], into the --help text for that flag, where it's readily visible. FossilOrigin-Name: c938e5d783b3f015b2a1b9f3711664b13497c4b71e4a890c65d6665539522ff8	2024-11-19 20:14:31 +00:00
stephan	c73b85c156	Get sqlite3_analyzer optionally linking against libsqlite3.so instead of embedding sqlite3.c. Patch mkccode.tcl to accept digits in its IFDEF/IFNDEF checks and sqlite3_analyzer.c.in to only include sqlite3.c if -DINCLUDE_SQLITE3_C is passed to mkccode.tcl. FossilOrigin-Name: 80f3bf8c2ee31ba1ab9187d64d5dcbbf97a61845a21b036f89ea9133153575c0	2024-11-19 19:47:51 +00:00
stephan	d3aaf963a9	Merge trunk into this branch. Fix a small jimtcl/tclsh regex incompatibility in mkccode.tcl. FossilOrigin-Name: 7fb3ebfec634e0508267049fddb2b513201dbefce4d378ca3ec261c5d8336d7f	2024-11-19 19:16:58 +00:00
drh	5608fb36ec	Enhancements to tool/mkccode.tcl such that it recognizes -D command line arguments and can use them in internal IFDEF and IFNDEF macros. Update the tool/sqlite3_analyzer.c.in script such that it omits the SQLite amalgamation if -DSQLITE_ENABLE_DBSTAT_VTAB is defined. FossilOrigin-Name: 7fec209290aa1a6dbbca8de154edaac5d8d0ce042bc0617d27fb2095c8d580f1	2024-11-19 18:26:47 +00:00
stephan	3cd93a0b67	Revert linking the shell to the dll for reasons explained in new makefile comments. Move the --with-debug configure flag into the developer options set and add commentary about why it should never be used for production builds. FossilOrigin-Name: 7b14309be42be4204c4d30e9741d56d75ab8ec34686791d032612337fe1c4dcf	2024-11-19 17:41:13 +00:00
stephan	5ef61df185	Reformulate sqldiff deps and rules so that the target matches the resulting file name, to avoid rebuilding it on every make invocation. Apply the same treatment to the sqlite3 CLI shell. FossilOrigin-Name: 9a17b83f859ef14629cb78d8c9af1b3f2493b0c8756bc2ebcf92f0872fb507c3	2024-11-19 16:40:49 +00:00
stephan	fbd9de1830	Initial configure support for linking certain binaries to libsqlite3.so instead of embedding it dynamically (which they typically do). This currently applies only to sqldiff, but support for adding the same for other tools is planned. This requires disabling the soname because setting it causes the build to link to whatever libsqlite3.so.0 is installed system-wide, so the soname now defaults to off. FossilOrigin-Name: 9192c146e2898456a6b8ea43a6f02c0227ddce5b584374fbeb2d63bd2ecd5d8c	2024-11-19 15:20:47 +00:00
dan	fefc7b1bf7	Fix a problem handling OOM errors in fts3 that could occur when parsing multi-token strings. FossilOrigin-Name: 4c4d1db00bd2c522165876dcf1606116a72525d9ffc891b266213704e25cde55	2024-11-19 11:58:32 +00:00
drh	ea206ad17b	Reduce divergence from begin-concurrent. FossilOrigin-Name: 84d41e9d04c62601e84621e8edb3b81abbacc366a6b5e7059f8b6796fade7ffa	2024-11-18 18:45:31 +00:00
drh	12599d566b	Attempt to reduce divergence with the wal2 branch. FossilOrigin-Name: e0d8f9916c2d899094b71af74f44e089a97db6d80f09eb80a5d8be733c1bf942	2024-11-18 17:05:45 +00:00
drh	5cad859f61	Attempt to reduce divergence with begin-concurrent. FossilOrigin-Name: f783d90187fb326faa3d0244b30138b023c4a8483486f260b7a022a927c13f3a	2024-11-18 16:44:26 +00:00
stephan	0df847cc77	Remove the .POSIX entries from the makefiles because they cause portability problems rather than solve them. FossilOrigin-Name: e1330ecf170436678c5c235557fb95b8acbf4fad62583edba4af03aff49afbc5	2024-11-18 14:15:08 +00:00
dan	de8aa17eb5	Fix a "applying zero offset to null pointer" usan error in the fts5 trigram tokenizer. FossilOrigin-Name: 9b79b999d4192a3a250bf343825f61e9cb83203e4d94dcfa20c320d5f1386d69	2024-11-18 14:08:42 +00:00
drh	9c8235b1bd	Fix minor problems in testrunner.tcl that pop up when the command-line arguments are such that no tests are run. FossilOrigin-Name: 321ded32f67550e964cd64d61aa0cbc0029ce2fdee4588a46b01dbb5aa87150b	2024-11-18 13:29:16 +00:00
drh	bbc6e5c9b7	Make the --vfstrace output from the CLI go to the same output channel as everything else. FossilOrigin-Name: f71d4900205ae6ee41f849c4026d0fe4d6cf281dfc3bac8105fc8e242d128b67	2024-11-17 11:42:43 +00:00
stephan	f6fdf32553	buildtclext.tcl: add docs explaining the DESTDIR check and break out of the auto_path search loop early if the DESTDIR is not writable, rather than re-checking that repeatedly. FossilOrigin-Name: d07be336eaeb9a5d56ac6e1b63f4d8e50d3ac236f5953cc178ef34542a7cd8fa	2024-11-17 09:49:53 +00:00
drh	70d390134c	Call fflush() on ".echo" output from the shell, so that the output to stdout is aligned with output to stderr. FossilOrigin-Name: c38b9db3c4f71706a7d211424da64311e6e5daf64b224565a6d82d4b1a68e261	2024-11-16 18:54:46 +00:00
drh	18689b8fb2	Fix argument expansion in sqlite-tclsh on Windows such that if an argument does not match a filename even after glob expansion, it is appended to the argument list verbatim. FossilOrigin-Name: cd942dce148c9d8f5a94cee61923aad8d1b732b807e004005f78323be30c02e7	2024-11-16 17:39:34 +00:00
stephan	50faa8d17e	Handle DESTDIR at an earlier phase in buildtclext.tcl to account for the is-writable-dir check and to filter out //zipfs: dirs as (im)possible installation targets. FossilOrigin-Name: d2e8c161a14fbbcc52c50dfd9274c9969e3c273e2cb7cbf2f865541af9f39ead	2024-11-16 17:09:55 +00:00
stephan	f6c7cc606e	Add the --destdir flag to the tclextension-install makefile target. FossilOrigin-Name: d1663cf05f7dcaafd479bacf083b6b774f34fd3db89012b49599d30817eb174f	2024-11-16 14:30:43 +00:00
stephan	d8f6222bee	Add --destdir flag support to buildtclext.tcl, but do not yet add that to the makefile (so that this change can be cherrypicked to the 3.47 build). FossilOrigin-Name: 67a3ca0c013b6a9da3b2c50ffc86b96b14454d1e45fa90d3b0a238488a783e79	2024-11-16 14:29:51 +00:00
stephan	4b24cb2a3a	Rename tool/tclConfigShToTcl.sh to tool/tclConfigShToAutoDef.sh in the name of pedantic correctness. FossilOrigin-Name: a7dd196d99ee7c75b92a03915f2b370e3f4c598eb8521b90af1cb5dd7f9ee49c	2024-11-16 10:42:33 +00:00
stephan	d676227f2c	configure: tiny simplification of proj-assert. FossilOrigin-Name: bfdc416b561937aa74483bf96c042e701e6f98997150f1f1a6b165cddd40d8d1	2024-11-16 09:00:31 +00:00
dan	67a3914e65	Merge latest trunk changes into this branch. FossilOrigin-Name: edb842349320eda9550bdfcd5a327949c5512e02f4b993782587b2131a425746	2024-11-06 17:31:48 +00:00
dan	8acaa6d039	Add tests for xInstToken() and prefix queries with various fts5 configurations. FossilOrigin-Name: 9cc04331a01760189d88697233009dbe8a60eda589792ad01b56300499e9f54d	2024-11-06 16:20:16 +00:00
dan	090b8649be	Allow an fts5 table or query to be configured to collect xInstToken data for any prefix terms as part of the first parse of the main index, if any. FossilOrigin-Name: 46929ae92b26f02bc70de9931b21a8a7cf9a2453d5fb07f68b712f62e28e9152	2024-11-02 19:10:50 +00:00
dan	32b4979559	Merge latest trunk changes into this branch. FossilOrigin-Name: 790c56d493c66a2136e24d349d169639809d70bfab6996975a403be568a267a5	2024-11-01 19:41:22 +00:00
dan	cd9b951e6b	Merge latest trunk changes into this branch. FossilOrigin-Name: bce9a524de6dda87daa90395cd55713d2d3ccfc090e53a947548e434db5eef5e	2024-10-28 10:39:14 +00:00
dan	b27a30c620	Merge latest trunk into this branch. FossilOrigin-Name: 740a37c5d54b57befa86a6bb299ffa89ed4243d10db885a08ab5c63238460dad	2024-10-27 14:41:58 +00:00
dan	d2a88e961a	Rationalize code further. And add tests. FossilOrigin-Name: 0ca002a4ab88f3e7ae1e6e518038157eaa20759f57888c2ed7e50cb92bd96348	2024-10-01 20:38:08 +00:00
dan	7d56669bc4	Rationalize some of the new code on this branch. FossilOrigin-Name: 66f209ba40e7de49b304d7931ff38a4994038452aab08e9347286a234c6f7075	2024-09-28 20:45:11 +00:00
dan	2eff8f2252	Change the way tokendata indexes are collected for prefix queries. FossilOrigin-Name: 204ddf4e726b695dd12ab4a945ec2461655aa0bcc38b74e970f07ed2ac43c6ff	2024-09-25 18:55:11 +00:00
dan	acfd1e2ba8	Merge trunk changes into this branch. FossilOrigin-Name: 9945206e6e26a48a49b9747650d299eb983cc21a3a61c621cd81f0bbc85a74d7	2024-09-24 15:43:52 +00:00
dan	e25a267e81	Fix the fts5 xInstToken() API for prefix queries that do not use prefix-indexes. This is experimental. FossilOrigin-Name: 97c2824f471e7e622c4a166947a6e8162cae891345101539829a6fcec83373fe	2024-09-14 20:30:14 +00:00