Change the automatic index mechanism so that it avoids creating transient

indexes on columns that are known to have low cardinality.

FossilOrigin-Name: 12ef3a8f3d1eb4bd1280f420cafc61c081942ad7

manifest

@@ -1,5 +1,5 @@
-C Fix\sissues\son\sunix\swith\sopening\sdatabase\sfiles\svia\ssymlinks\sthat\sare\snot\sin\sthe\scurrent\sworking\sdirectory.\sAnd\swith\snested\ssymlinks.
-D 2016-01-26T14:48:02.511
+C Change\sthe\sautomatic\sindex\smechanism\sso\sthat\sit\savoids\screating\stransient\nindexes\son\scolumns\sthat\sare\sknown\sto\shave\slow\scardinality.
+D 2016-01-26T15:23:58.702
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6fca5455aaecbd14479f33f091aa19df2d3d2969
@@ -283,7 +283,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c 9d649e46c780166e416fb11dbd23f8d49aab8267
-F src/analyze.c 0043d3e501f04297fed2bb50b488bc08d5c39f36
+F src/analyze.c dfc207500538c0f2c59217c1ed5c5376080438e9
 F src/attach.c 07b3a34a1702dce92a7f1d3888c0c06222b63760
 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 2869a76c03eb393ee795416e2387005553df72bc
@@ -292,7 +292,7 @@ F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73
 F src/btree.c f224ae877fde69d1a9d430f502edaf8502752dbe
 F src/btree.h 526137361963e746949ab966a910c7f455ac6b04
 F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5
-F src/build.c b4eba1e84752ec9cae7ff3dacd5a8b6d1ab8deb9
+F src/build.c 9135b9b8c355abb3b347cf435315ecdfe912b78a
 F src/callback.c 29ae4faba226c7ebb9aee93016b5ce8a8f071261
 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f
 F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198
@@ -351,7 +351,7 @@ F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4
 F src/sqlite.h.in 214476a62012e578f42133a9a3b4f97a9aa421a3
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
-F src/sqliteInt.h 74e10a74116df0aec9d4a3e134f1a86cc34c2f14
+F src/sqliteInt.h 13058a77641cf402331177fd5c0396b2ad1ade54
 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
@@ -425,7 +425,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c af9bf5dcec1a0e52726c550924aa91d837166251
+F src/where.c 3e70fe90129ed22a603b038ae2e499adb8d0e33f
 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a
 F src/wherecode.c 8dee26eb181ea9daa8b1a4d96f34c0860aaf99bd
 F src/whereexpr.c 197a448b52073aee43eca3a2233fc113369eb2d4
@@ -1421,8 +1421,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P abd2b357c55fdcdbc8e66a81a3fbb7f822ea0a4f 4a4385564dd3887a7953820b60c99d6ce289f96a
-R 1d2d15b294b146d0b5d200f0cf62495d
-T +closed 4a4385564dd3887a7953820b60c99d6ce289f96a
-U dan
-Z 7fd087e8bb59761ded2cdf6adb6cd578
+P 4003db4a49c6b623750e56f626fa492c8402067f
+R df25e6936324868b02b751b577e13e17
+T *branch * autoindex-planning
+T *sym-autoindex-planning *
+T -sym-trunk *
+U drh
+Z 69366cc7d5918ba0816a4dae1ba05b53

manifest.uuid

@@ -1 +1 @@
-4003db4a49c6b623750e56f626fa492c8402067f
+12ef3a8f3d1eb4bd1280f420cafc61c081942ad7

src/analyze.c

@@ -1529,7 +1529,19 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){
 #endif
     pIndex->bUnordered = 0;
     decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex);
-    if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0];
+    if( pIndex->pPartIdxWhere==0 ){
+      int i, j;
+      /* TUNING: Any column that cannot narrow down the number of table rows
+      ** to less than 30 should not be considered as a column for use in
+      ** an automatic index */
+      for(i=0; i<nCol; i++){
+        assert( 49==sqlite3LogEst(30) );
+        if( aiRowEst[i+1]>49 && (j = pIndex->aiColumn[i])>=0 ){
+          pTable->aCol[j].colFlags |= COLFLAG_NOAUTO;
+        }
+      }
+      pTable->nRowLogEst = pIndex->aiRowLogEst[0];
+    }
   }else{
     Index fakeIdx;
     fakeIdx.szIdxRow = pTable->szTabRow;

src/build.c

@@ -3153,8 +3153,16 @@ Index *sqlite3CreateIndex(
       assert( j<=0x7fff );
       if( j<0 ){
         j = pTab->iPKey;
-      }else if( pTab->aCol[j].notNull==0 ){
-        pIndex->uniqNotNull = 0;
+      }else{
+        if( pTab->aCol[j].notNull==0 ){
+          pIndex->uniqNotNull = 0;
+        }
+        if( i==0 && pPIWhere==0 ){
+          /* Exclude the left-most column of every complete index from
+          ** consideration as an automatic index term.  Better to use the real
+          ** index */
+          pTab->aCol[j].colFlags |= COLFLAG_NOAUTO;
+        }
       }
       pIndex->aiColumn[i] = (i16)j;
     }

src/sqliteInt.h

@@ -1540,6 +1540,7 @@ struct Column {
 */
 #define COLFLAG_PRIMKEY  0x0001    /* Column is part of the primary key */
 #define COLFLAG_HIDDEN   0x0002    /* A hidden column in a virtual table */
+#define COLFLAG_NOAUTO   0x0004    /* Not useful in an automatic index */
 
 /*
 ** A "Collating Sequence" is defined by an instance of the following

src/where.c

@@ -578,15 +578,19 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){
 static int termCanDriveIndex(
   WhereTerm *pTerm,              /* WHERE clause term to check */
   struct SrcList_item *pSrc,     /* Table we are trying to access */
-  Bitmask notReady               /* Tables in outer loops of the join */
+  Bitmask notReady,              /* Tables in outer loops of the join */
+  int excludeNoauto              /* Answer FALSE if pTerm is COLFLAG_NOAUTO */
 ){
   char aff;
+  Column *pCol;
   if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
   if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
   if( (pTerm->prereqRight & notReady)!=0 ) return 0;
   if( pTerm->u.leftColumn<0 ) return 0;
-  aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity;
+  pCol = &pSrc->pTab->aCol[pTerm->u.leftColumn];
+  aff = pCol->affinity;
   if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
+  if( excludeNoauto && (pCol->colFlags & COLFLAG_NOAUTO)!=0 ) return 0;
   testcase( pTerm->pExpr->op==TK_IS );
   return 1;
 }
@@ -655,7 +659,7 @@ static void constructAutomaticIndex(
       pPartial = sqlite3ExprAnd(pParse->db, pPartial,
                                 sqlite3ExprDup(pParse->db, pExpr, 0));
     }
-    if( termCanDriveIndex(pTerm, pSrc, notReady) ){
+    if( termCanDriveIndex(pTerm, pSrc, notReady, 0) ){
       int iCol = pTerm->u.leftColumn;
       Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
       testcase( iCol==BMS );
@@ -708,7 +712,7 @@ static void constructAutomaticIndex(
   n = 0;
   idxCols = 0;
   for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
-    if( termCanDriveIndex(pTerm, pSrc, notReady) ){
+    if( termCanDriveIndex(pTerm, pSrc, notReady, 0) ){
       int iCol = pTerm->u.leftColumn;
       Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
       testcase( iCol==BMS-1 );
@@ -2627,7 +2631,7 @@ static int whereLoopAddBtree(
     WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
     for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
       if( pTerm->prereqRight & pNew->maskSelf ) continue;
-      if( termCanDriveIndex(pTerm, pSrc, 0) ){
+      if( termCanDriveIndex(pTerm, pSrc, 0, 1) ){
         pNew->u.btree.nEq = 1;
         pNew->nSkip = 0;
         pNew->u.btree.pIndex = 0;