Have SQLite request a plan with no setup-cost from xBestIndex if the plan in question may only be used as the outermost loop of the query.

FossilOrigin-Name: c3740ef6e4d1301fa69f925721faf7e7fdd7cfa902ad2cf90d6e72a38d6a5c08

manifest

@@ -1,5 +1,5 @@
-C Enhance\sthe\s"mktoolzip.tcl"\sscript\s(used\sby\sthe\s"tool-zip"\smakefile\starget)\sso\sthat\nit\suses\s"fossil\stest-filezip"\sto\sbuild\sthe\sZIP\sarchive\sif\savailable,\sand\sso\sthat\nit\salso\sgenerates\swindows-dll-*.zip\son\sWindows\splatforms.
-D 2025-07-14T16:05:59.333
+C Have\sSQLite\srequest\sa\splan\swith\sno\ssetup-cost\sfrom\sxBestIndex\sif\sthe\splan\sin\squestion\smay\sonly\sbe\sused\sas\sthe\soutermost\sloop\sof\sthe\squery.
+D 2025-07-15T17:03:51.141
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -788,7 +788,7 @@ F src/resolve.c d3ee7ed308d46f4ee6d3bb6316d8d6f87158f93a7fd616732138cc953cf364f0
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c fc003cad96a105765261f7b6c5f4596e505894262bb5593cb29e10b682800d12
 F src/shell.c.in 73c0eeb7c265d59b99219d5aa055f412f07842088d8036b6d259927d85dd1bbf
-F src/sqlite.h.in 5c54f2461a1ea529bab8499148a2b238e2d4bb571d59e8ea5322d0c190abb693
+F src/sqlite.h.in 65939c73f3a0fa9cd2da409a1cc27b3abbf758621736c5ce8030dc072bf8affa
 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479
 F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e
 F src/sqliteInt.h 984e80f6cfdb1cd65ab465d14d82b6b56af4f7dc308c4bc1da85074a5b4ca755
@@ -807,7 +807,7 @@ F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0
 F src/test9.c df9ddc7db6ef1b8cf745866ee229090779728bcbe660c7f297d3127ab21d92af
 F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f
 F src/test_backup.c a2bfd90d2ff2511b8635507bdb30fa9b605ade19c16b533066cae3077f5bdb72
-F src/test_bestindex.c 3401bee51665cbf7f9ed2552b5795452a8b86365e4c9ece745b54155a55670c6
+F src/test_bestindex.c 2d158de210513ee05b500502325558996788d7c19fbb4e27bfcde0f69976dfbe
 F src/test_blob.c 77b994e17f2c87055f44fd96c9a206c5a7155bae2cda2769af60c2f3582f962c
 F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5
 F src/test_config.c 7f412406592794636d6226268e26d413850a9f799bc5f3c01afc2820b165fca8
@@ -868,7 +868,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 20be6f0a25a80b7897cf2a5369bfd37ef198e6f0b6cdef16d83eee856056b159
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
-F src/where.c 6a9266dd1a559d48d8c7ca670a3e80143c7913153f7d1ceb0a4eca1087318951
+F src/where.c 7fab63d9b986ee9d6a1c496556d2afcb6435d81a9f3e75faaf8f2810ea7eb44a
 F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da
 F src/wherecode.c 2a2d2993fd98c46f525f71b3bfd330fde73d8613aa0ff3e20402dd1fc63470af
 F src/whereexpr.c 0a7fe115adad30def38aeab6ac1d35fb67782cee92a43df7448136240accd4dd
@@ -965,6 +965,7 @@ F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572e
 F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce
 F test/bestindexC.test 95b4a527b1a5d07951d731604a6d4cf7e5a806b39cea0e7819d4c9667e11c3fc
 F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3
+F test/bestindexE.test f92bc0c4eaf285ce3fe771a9a7c8e3980aa054bb2391945315b55e4d2dbf42c6
 F test/between.test b9a65fb065391980119e8a781a7409d3fcf059d89968279c750e190a9a1d5263
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
@@ -2213,8 +2214,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 3c0afda372cd688e9aff7be1c0a41f557c8c8819f5f47420166ce8255cd806c8
-R e290ca399c8e663ed9c1c500434c64ea
-U drh
-Z 55552671916a227c18ab9a99dcc87aef
+P f3fdcfa1834411ea5dcb6bbc260c50e304feb50acebbd85a4fe554da797be7d5
+R 0fa274d8c077fb8b9e746584ff99fb65
+U dan
+Z d274fd77be688740fb7252a55879eb6c
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-63ddc215eb6106985c53d3c8d99f5445a92acf4359a6bb2681882c49583df871
+c3740ef6e4d1301fa69f925721faf7e7fdd7cfa902ad2cf90d6e72a38d6a5c08

src/sqlite.h.in

@@ -7633,6 +7633,31 @@ struct sqlite3_module {
 ** ^The estimatedRows value is an estimate of the number of rows that
 ** will be returned by the strategy.
 **
+** The estimatedSetup field is used to specify the estimated one-time setup
+** cost of a query strategy, which is typically incurred during the first 
+** call to xFilter. In most cases, this value should be zero. However, if 
+** the strategy requires initialization of an internal index or data 
+** structure - similar to SQLite's native [automatic indexes] - a non-zero 
+** value may be used to reflect that overhead.
+**
+** A negative value for estimatedSetup is treated by SQLite as zero.
+**
+** When the xBestIndex method is invoked, estimatedSetup may be initialized
+** to either 0.0 or to a negative value. If it is set to a negative value,
+** then SQLite is requesting a strategy with no (or minimal) setup cost. The
+** virtual table implementation does not have to honor the request, but
+** it may be more efficient to do so.
+**
+** In practice, xBestIndex is usually invoked with estimatedSetup initialized
+** to 0.0. If the implementation then returns a strategy with estimatedSetup
+** set to a value greater than 0.0, then SQLite invokes the xBestIndex method 
+** a second time with the same parameters, except this time with estimatedSetup
+** initialized to a negative value. This allows the virtual table to present 
+** two different strategies - one with a higher setup cost and potentially
+** lower per-row cost, and another with minimal setup cost - enabling the 
+** query planner to make an informed decision based on query-specific 
+** tradeoffs.
+**
 ** The xBestIndex method may optionally populate the idxFlags field with a
 ** mask of SQLITE_INDEX_SCAN_* flags. One such flag is
 ** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN]
@@ -7686,13 +7711,15 @@ struct sqlite3_index_info {
   char *idxStr;              /* String, possibly obtained from sqlite3_malloc */
   int needToFreeIdxStr;      /* Free idxStr using sqlite3_free() if true */
   int orderByConsumed;       /* True if output is already ordered */
-  double estimatedCost;           /* Estimated cost of using this index */
+  double estimatedCost;      /* Estimated per-scan cost of using this index */
   /* Fields below are only available in SQLite 3.8.2 and later */
   sqlite3_int64 estimatedRows;    /* Estimated number of rows returned */
   /* Fields below are only available in SQLite 3.9.0 and later */
   int idxFlags;              /* Mask of SQLITE_INDEX_SCAN_* flags */
   /* Fields below are only available in SQLite 3.10.0 and later */
   sqlite3_uint64 colUsed;    /* Input: Mask of columns used by statement */
+  /* Fields below are only available in SQLite 3.51.0 and later */
+  double estimatedSetup;     /* Estimated setup cost of this index */
 };
 
 /*

src/test_bestindex.c

@@ -525,6 +525,7 @@ static int SQLITE_TCLAPI testBestIndexObj(
     "in",                         /* 4 */
     "rhs_value",                  /* 5 */
     "collation",                  /* 6 */
+    "estimatedSetup",             /* 7 */
     0
   };
   int ii;
@@ -550,6 +551,10 @@ static int SQLITE_TCLAPI testBestIndexObj(
     Tcl_WrongNumArgs(interp, 2, objv, "INDEX ?DEFAULT?");
     return TCL_ERROR;
   }
+  if( ii==7 && objc!=2 ){
+    Tcl_WrongNumArgs(interp, 2, objv, "");
+    return TCL_ERROR;
+  }
 
   switch( ii ){
     case 0: assert( sqlite3_stricmp(azSub[ii], "constraints")==0 );
@@ -616,6 +621,11 @@ static int SQLITE_TCLAPI testBestIndexObj(
       Tcl_SetObjResult(interp, Tcl_NewStringObj(zColl, -1));
       break;
     }
+
+    case 7: assert( sqlite3_stricmp(azSub[ii], "estimatedSetup")==0 ); {
+      Tcl_SetObjResult(interp, Tcl_NewDoubleObj(pIdxInfo->estimatedSetup));
+      break;
+    }
   }
 
   return TCL_OK;
@@ -675,6 +685,9 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
         if( sqlite3_stricmp("cost", zCmd)==0 ){
           rc = Tcl_GetDoubleFromObj(interp, p, &pIdxInfo->estimatedCost);
         }else
+        if( sqlite3_stricmp("setup", zCmd)==0 ){
+          rc = Tcl_GetDoubleFromObj(interp, p, &pIdxInfo->estimatedSetup);
+        }else
         if( sqlite3_stricmp("orderby", zCmd)==0 ){
           rc = Tcl_GetIntFromObj(interp, p, &pIdxInfo->orderByConsumed);
         }else

src/where.c

@@ -4250,7 +4250,7 @@ static int allConstraintsUsed(
 ** Output parameter *pbIn is set to true if the plan added to pBuilder
 ** uses one or more WO_IN terms, or false otherwise.
 */
-static int whereLoopAddVirtualOne(
+static int whereLoopAddVirtualPlan(
   WhereLoopBuilder *pBuilder,
   Bitmask mPrereq,                /* Mask of tables that must be used. */
   Bitmask mUsable,                /* Mask of usable tables */
@@ -4408,7 +4408,7 @@ static int whereLoopAddVirtualOne(
   pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
       pIdxInfo->nOrderBy : 0);
   pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0;
-  pNew->rSetup = 0;
+  pNew->rSetup = sqlite3LogEstFromDouble(pIdxInfo->estimatedSetup);
   pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
   pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
 
@@ -4431,6 +4431,59 @@ static int whereLoopAddVirtualOne(
   return rc;
 }
 
+/*
+** This is a wrapper around whereLoopAddVirtualPlan(). All the arguments
+** passed to this function are passed directly through to
+** whereLoopAddVirtualPlan(). 
+**
+** This wrapper always invokes whereLoopAddVirtualPlan() at least once.
+** If that invocation returns a plan with a non-zero setup cost, it is
+** then invoked again with the same parameters, except with estimatedSetup
+** initialized to -1.0 (instead of 0.0) to indicate to the vtab implementation
+** that SQLite is requesting a plan with no setup cost.
+**
+** Output variable (*pbIn) is set to true if the final call to
+** whereLoopAddVirtualPlan() made by this function returns a plan that uses
+** multiple xFilter() calls for an IN(...) constraint.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+static int whereLoopAddVirtualOne(
+  WhereLoopBuilder *pBuilder,
+  Bitmask mPrereq,                /* Mask of tables that must be used. */
+  Bitmask mUsable,                /* Mask of usable tables */
+  u16 mExclude,                   /* Exclude terms using these operators */
+  sqlite3_index_info *pIdxInfo,   /* Populated object for xBestIndex */
+  u16 mNoOmit,                    /* Do not omit these constraints */
+  int *pbIn,                      /* OUT: True if plan uses an IN(...) op */
+  int *pbRetryLimit               /* OUT: Retry without LIMIT/OFFSET */
+){
+  int rc;
+
+  /* If this is guaranteed to be the outermost table in the join, either
+  ** because it is the only table in the join or because no prereq tables
+  ** are allowed, then request a plan with no setup-cost by setting
+  ** estimatedSetup to a negative value.  */
+  pIdxInfo->estimatedSetup = -1.0;
+  if( (mUsable|mPrereq) && pBuilder->pWInfo->pTabList->nSrc>1 ){
+    pIdxInfo->estimatedSetup = 0.0;
+  }
+
+  rc = whereLoopAddVirtualPlan(pBuilder, 
+      mPrereq, mUsable, mExclude, pIdxInfo, mNoOmit, pbIn, pbRetryLimit
+  );
+
+  /* If the xBestIndex() method returned a plan with a setup-cost, request
+  ** one with a zero setup-cost as well.  */
+  if( rc==SQLITE_OK && pIdxInfo->estimatedSetup>0.0 ){
+    pIdxInfo->estimatedSetup = -1.0;
+    rc = whereLoopAddVirtualPlan(pBuilder, 
+        mPrereq, mUsable, mExclude, pIdxInfo, mNoOmit, pbIn, pbRetryLimit
+    );
+  }
+  return rc;
+}
+
 /*
 ** Return the collating sequence for a constraint passed into xBestIndex.
 **

test/bestindexE.test

@@ -0,0 +1,154 @@
+# 2025-07-14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# This file contains tests focused on virtual tables that set non-zero
+# sqlite3_index_info.estimatedSetup values from within xBestIndex.
+#
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindexE
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+register_tcl_module db
+
+proc vtab_command {tbl method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE x(tid, data)"
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+
+      set estSetup [$hdl estimatedSetup]
+      set iTidEq -1
+
+      #puts "estSetup is $estSetup"
+
+      set iCons 0
+      foreach c [$hdl constraints] {
+        array set C $c
+        if {$C(usable) && $C(op)=="eq" && $C(column)==0} {
+          set iTidEq $iCons
+        }
+        incr iCons
+      }
+
+      set nRow [db one "SELECT count(*) FROM $tbl"]
+      if {$tbl=="ee"} {
+        # Events table. Three strategies: 
+        #
+        #    1) Linear scan
+        #    2) Scan with tid=? constraint
+        #    3) Auto index on tid=? constraint
+        #
+        if {$iTidEq>=0} {
+          if {$estSetup<0.0} {
+            set ret "cost $nRow rows 1 idxStr by-rowid omit $iTidEq"
+          } else {
+            set ret "cost 10.0 rows 1 idxStr autoindex setup [expr $nRow*10] omit $iTidEq"
+          }
+        } else {
+          set ret "cost $nRow rows $nRow idxStr linear"
+        }
+
+      } else {
+        # Threads table. Two strategies: linear scan or fast lookup by rowid.
+        if {$iTidEq>=0} {
+          set ret "cost 10.0 rows 1 omit $iTidEq idxStr by-rowid"
+        } else {
+          set ret "cost $nRow rows $nRow idxStr linear"
+        }
+      }
+
+      return $ret
+    }
+
+    xFilter {
+      return "sql {SELECT a, * FROM $tbl}"
+    }
+  }
+
+  return {}
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE ee(a INTEGER PRIMARY KEY, e);
+  CREATE TABLE tt(a INTEGER PRIMARY KEY, t);
+
+  CREATE VIRTUAL TABLE event USING tcl(vtab_command ee);
+  CREATE VIRTUAL TABLE thread USING tcl(vtab_command tt);
+} {}
+
+proc insert_data {nThread nEvent} {
+  set nEvent [expr $nEvent]
+  set nThread [expr $nThread]
+
+  execsql {
+    DELETE FROM ee;
+    DELETE FROM tt;
+    WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nEvent
+    )
+    INSERT INTO ee(e) SELECT hex(randomblob(10)) FROM s;
+
+    WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nThread
+    )
+    INSERT INTO tt(t) SELECT hex(randomblob(10)) FROM s;
+  }
+}
+
+#set sqlite_where_trace [expr 0x01]
+#puts [query_plan_graph "SELECT * FROM event e, thread t WHERE e.tid = t.tid;"]
+
+# Cost = (nThread + 10*nEvent + 10*nEvent)
+#
+insert_data 100 100000
+do_eqp_test 1.1 {
+  SELECT * FROM event e, thread t WHERE e.tid = t.tid;
+} {
+  QUERY PLAN
+  |--SCAN t VIRTUAL TABLE INDEX 0:linear
+  `--SCAN e VIRTUAL TABLE INDEX 0:autoindex
+}
+
+# Cost = (nThread + nThread * nEvent)
+#
+insert_data 3 100000
+do_eqp_test 1.2 {
+  SELECT * FROM event e, thread t WHERE e.tid = t.tid;
+} {
+  QUERY PLAN
+  |--SCAN t VIRTUAL TABLE INDEX 0:linear
+  `--SCAN e VIRTUAL TABLE INDEX 0:by-rowid
+}
+
+# Cost = nEvent + nEvent * 10
+#
+insert_data 1000 10000
+do_eqp_test 1.3 {
+  SELECT * FROM event e, thread t WHERE e.tid = t.tid;
+} {
+  QUERY PLAN
+  |--SCAN e VIRTUAL TABLE INDEX 0:linear
+  `--SCAN t VIRTUAL TABLE INDEX 0:by-rowid
+}
+
+
+
+finish_test
+
+