Minor size and performance improvements to the push-down optimization.

FossilOrigin-Name: 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d
Within a loop that uses a non-covering index test, test non-indexed terms that
2017-04-29 14:56:55 +00:00 · 2017-04-28 19:59:55 +00:00 · 2017-04-26 17:21:33 +00:00 · 2017-04-26 14:34:04 +00:00 · 2017-04-26 04:32:17 +00:00 · 2017-04-24 14:16:55 +00:00
54 changed files with 1905 additions and 594 deletions
@@ -21,7 +21,7 @@ USE_AMALGAMATION = 1
 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
 !IFNDEF USE_FULLWARN
-USE_FULLWARN = 0
+USE_FULLWARN = 1
 !ENDIF

 # Set this non-0 to enable treating warnings as errors (-WX, etc) when
@@ -372,8 +372,8 @@ int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
 }

 /*
-** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
-** 32-bit integer before it is returned.
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to 
+** a non-negative 32-bit integer before it is returned.
 */
 int sqlite3Fts3GetVarint32(const char *p, int *pi){
  u32 a;
@@ -389,7 +389,9 @@ int sqlite3Fts3GetVarint32(const char *p, int *pi){
  GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
  GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
  a = (a & 0x0FFFFFFF );
-  *pi = (int)(a | ((u32)(*p & 0x0F) << 28));
+  *pi = (int)(a | ((u32)(*p & 0x07) << 28));
+  assert( 0==(a & 0x80000000) );
+  assert( *pi>=0 );
  return 5;
 }

@@ -1219,65 +1221,66 @@ static int fts3InitVtab(
            break;
          }
        }
-        if( iOpt==SizeofArray(aFts4Opt) ){
-          sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
-          rc = SQLITE_ERROR;
-        }else{
-          switch( iOpt ){
-            case 0:               /* MATCHINFO */
-              if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
-                sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
-                rc = SQLITE_ERROR;
-              }
-              bNoDocsize = 1;
-              break;
+        switch( iOpt ){
+          case 0:               /* MATCHINFO */
+            if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
+              sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
+              rc = SQLITE_ERROR;
+            }
+            bNoDocsize = 1;
+            break;

-            case 1:               /* PREFIX */
-              sqlite3_free(zPrefix);
-              zPrefix = zVal;
-              zVal = 0;
-              break;
+          case 1:               /* PREFIX */
+            sqlite3_free(zPrefix);
+            zPrefix = zVal;
+            zVal = 0;
+            break;

-            case 2:               /* COMPRESS */
-              sqlite3_free(zCompress);
-              zCompress = zVal;
-              zVal = 0;
-              break;
+          case 2:               /* COMPRESS */
+            sqlite3_free(zCompress);
+            zCompress = zVal;
+            zVal = 0;
+            break;

-            case 3:               /* UNCOMPRESS */
-              sqlite3_free(zUncompress);
-              zUncompress = zVal;
-              zVal = 0;
-              break;
+          case 3:               /* UNCOMPRESS */
+            sqlite3_free(zUncompress);
+            zUncompress = zVal;
+            zVal = 0;
+            break;

-            case 4:               /* ORDER */
-              if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
-               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
-              ){
-                sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
-                rc = SQLITE_ERROR;
-              }
-              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
-              break;
+          case 4:               /* ORDER */
+            if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
+             && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
+            ){
+              sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
+              rc = SQLITE_ERROR;
+            }
+            bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
+            break;

-            case 5:              /* CONTENT */
-              sqlite3_free(zContent);
-              zContent = zVal;
-              zVal = 0;
-              break;
+          case 5:              /* CONTENT */
+            sqlite3_free(zContent);
+            zContent = zVal;
+            zVal = 0;
+            break;

-            case 6:              /* LANGUAGEID */
-              assert( iOpt==6 );
-              sqlite3_free(zLanguageid);
-              zLanguageid = zVal;
-              zVal = 0;
-              break;
+          case 6:              /* LANGUAGEID */
+            assert( iOpt==6 );
+            sqlite3_free(zLanguageid);
+            zLanguageid = zVal;
+            zVal = 0;
+            break;

-            case 7:              /* NOTINDEXED */
-              azNotindexed[nNotindexed++] = zVal;
-              zVal = 0;
-              break;
-          }
+          case 7:              /* NOTINDEXED */
+            azNotindexed[nNotindexed++] = zVal;
+            zVal = 0;
+            break;
+
+          default:
+            assert( iOpt==SizeofArray(aFts4Opt) );
+            sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
+            rc = SQLITE_ERROR;
+            break;
        }
        sqlite3_free(zVal);
      }
@@ -1846,7 +1849,8 @@ static int fts3ScanInteriorNode(
    isFirstTerm = 0;
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
-    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+    assert( nPrefix>=0 && nSuffix>=0 );
+    if( &zCsr[nSuffix]>zEnd ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
@@ -2656,7 +2660,7 @@ int sqlite3Fts3FirstFilter(
    fts3ColumnlistCopy(0, &p);
  }

-  while( p<pEnd && *p==0x01 ){
+  while( p<pEnd ){
    sqlite3_int64 iCol;
    p++;
    p += sqlite3Fts3GetVarint(p, &iCol);
@@ -3336,33 +3340,38 @@ static int fts3ColumnMethod(
  /* The column value supplied by SQLite must be in range. */
  assert( iCol>=0 && iCol<=p->nColumn+2 );

-  if( iCol==p->nColumn+1 ){
-    /* This call is a request for the "docid" column. Since "docid" is an 
-    ** alias for "rowid", use the xRowid() method to obtain the value.
-    */
-    sqlite3_result_int64(pCtx, pCsr->iPrevId);
-  }else if( iCol==p->nColumn ){
-    /* The extra column whose name is the same as the table.
-    ** Return a blob which is a pointer to the cursor.  */
-    sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
-  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
-    sqlite3_result_int64(pCtx, pCsr->iLangid);
-  }else{
-    /* The requested column is either a user column (one that contains 
-    ** indexed data), or the language-id column.  */
-    rc = fts3CursorSeek(0, pCsr);
+  switch( iCol-p->nColumn ){
+    case 0:
+      /* The special 'table-name' column */
+      sqlite3_result_blob(pCtx, &pCsr, sizeof(Fts3Cursor*), SQLITE_TRANSIENT);
+      sqlite3_result_subtype(pCtx, SQLITE_BLOB);
+      break;

-    if( rc==SQLITE_OK ){
-      if( iCol==p->nColumn+2 ){
-        int iLangid = 0;
-        if( p->zLanguageid ){
-          iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1);
-        }
-        sqlite3_result_int(pCtx, iLangid);
-      }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
+    case 1:
+      /* The docid column */
+      sqlite3_result_int64(pCtx, pCsr->iPrevId);
+      break;
+
+    case 2:
+      if( pCsr->pExpr ){
+        sqlite3_result_int64(pCtx, pCsr->iLangid);
+        break;
+      }else if( p->zLanguageid==0 ){
+        sqlite3_result_int(pCtx, 0);
+        break;
+      }else{
+        iCol = p->nColumn;
+        /* fall-through */
+      }
+
+    default:
+      /* A user column. Or, if this is a full-table scan, possibly the
+      ** language-id column. Seek the cursor. */
+      rc = fts3CursorSeek(0, pCsr);
+      if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){
        sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
      }
-    }
+      break;
  }

  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
@@ -3442,17 +3451,11 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
 static int fts3SetHasStat(Fts3Table *p){
  int rc = SQLITE_OK;
  if( p->bHasStat==2 ){
-    const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'";
-    char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName);
-    if( zSql ){
-      sqlite3_stmt *pStmt = 0;
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-      if( rc==SQLITE_OK ){
-        int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
-        rc = sqlite3_finalize(pStmt);
-        if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat;
-      }
-      sqlite3_free(zSql);
+    char *zTbl = sqlite3_mprintf("%s_stat", p->zName);
+    if( zTbl ){
+      int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0);
+      sqlite3_free(zTbl);
+      p->bHasStat = (res==SQLITE_OK);
    }else{
      rc = SQLITE_NOMEM;
    }
@@ -3559,18 +3562,16 @@ static int fts3FunctionArg(
  sqlite3_value *pVal,            /* argv[0] passed to function */
  Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
 ){
-  Fts3Cursor *pRet;
-  if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
-   || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
-  ){
+  int rc = SQLITE_OK;
+  if( sqlite3_value_subtype(pVal)==SQLITE_BLOB ){
+    *ppCsr = *(Fts3Cursor**)sqlite3_value_blob(pVal);
+  }else{
    char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
    sqlite3_result_error(pContext, zErr, -1);
    sqlite3_free(zErr);
-    return SQLITE_ERROR;
+    rc = SQLITE_ERROR;
  }
-  memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
-  *ppCsr = pRet;
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -3957,7 +3958,7 @@ int sqlite3Fts3Init(sqlite3 *db){
 #endif

  /* Create the virtual table wrapper around the hash-table and overload 
-  ** the two scalar functions. If this is successful, register the
+  ** the four scalar functions. If this is successful, register the
  ** module with sqlite.
  */
  if( SQLITE_OK==rc 
@@ -4540,7 +4541,7 @@ static int fts3EvalIncrPhraseNext(
  ** one incremental token. In which case the bIncr flag is set. */
  assert( p->bIncr==1 );

-  if( p->nToken==1 && p->bIncr ){
+  if( p->nToken==1 ){
    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
        &pDL->iDocid, &pDL->pList, &pDL->nList
    );
@@ -4773,6 +4774,7 @@ static void fts3EvalTokenCosts(
 ** the number of overflow pages consumed by a record B bytes in size.
 */
 static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
+  int rc = SQLITE_OK;
  if( pCsr->nRowAvg==0 ){
    /* The average document size, which is required to calculate the cost
    ** of each doclist, has not yet been determined. Read the required 
@@ -4812,11 +4814,10 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
    pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
    assert( pCsr->nRowAvg>0 ); 
    rc = sqlite3_reset(pStmt);
-    if( rc!=SQLITE_OK ) return rc;
  }

  *pnPage = pCsr->nRowAvg;
-  return SQLITE_OK;
+  return rc;
 }

 /*
@@ -5166,7 +5167,8 @@ static void fts3EvalNextRow(
          pExpr->iDocid = pLeft->iDocid;
          pExpr->bEof = (pLeft->bEof || pRight->bEof);
          if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){
-            if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){
+            assert( pRight->eType==FTSQUERY_PHRASE );
+            if( pRight->pPhrase->doclist.aAll ){
              Fts3Doclist *pDl = &pRight->pPhrase->doclist;
              while( *pRc==SQLITE_OK && pRight->bEof==0 ){
                memset(pDl->pList, 0, pDl->nList);
@@ -5195,7 +5197,7 @@ static void fts3EvalNextRow(

        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
          fts3EvalNextRow(pCsr, pLeft, pRc);
-        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
+        }else if( pLeft->bEof || iCmp>0 ){
          fts3EvalNextRow(pCsr, pRight, pRc);
        }else{
          fts3EvalNextRow(pCsr, pLeft, pRc);
@@ -5287,7 +5289,6 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
  */
  if( *pRc==SQLITE_OK 
   && pExpr->eType==FTSQUERY_NEAR 
-   && pExpr->bEof==0
   && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  ){
    Fts3Expr *p; 
@@ -5296,42 +5297,39 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){

    /* Allocate temporary working space. */
    for(p=pExpr; p->pLeft; p=p->pLeft){
+      assert( p->pRight->pPhrase->doclist.nList>0 );
      nTmp += p->pRight->pPhrase->doclist.nList;
    }
    nTmp += p->pPhrase->doclist.nList;
-    if( nTmp==0 ){
+    aTmp = sqlite3_malloc(nTmp*2);
+    if( !aTmp ){
+      *pRc = SQLITE_NOMEM;
      res = 0;
    }else{
-      aTmp = sqlite3_malloc(nTmp*2);
-      if( !aTmp ){
-        *pRc = SQLITE_NOMEM;
-        res = 0;
-      }else{
-        char *aPoslist = p->pPhrase->doclist.pList;
-        int nToken = p->pPhrase->nToken;
+      char *aPoslist = p->pPhrase->doclist.pList;
+      int nToken = p->pPhrase->nToken;

-        for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
-          Fts3Phrase *pPhrase = p->pRight->pPhrase;
-          int nNear = p->nNear;
-          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
-        }
-
-        aPoslist = pExpr->pRight->pPhrase->doclist.pList;
-        nToken = pExpr->pRight->pPhrase->nToken;
-        for(p=pExpr->pLeft; p && res; p=p->pLeft){
-          int nNear;
-          Fts3Phrase *pPhrase;
-          assert( p->pParent && p->pParent->pLeft==p );
-          nNear = p->pParent->nNear;
-          pPhrase = (
-              p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
-              );
-          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
-        }
+      for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
+        Fts3Phrase *pPhrase = p->pRight->pPhrase;
+        int nNear = p->nNear;
+        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
      }

-      sqlite3_free(aTmp);
+      aPoslist = pExpr->pRight->pPhrase->doclist.pList;
+      nToken = pExpr->pRight->pPhrase->nToken;
+      for(p=pExpr->pLeft; p && res; p=p->pLeft){
+        int nNear;
+        Fts3Phrase *pPhrase;
+        assert( p->pParent && p->pParent->pLeft==p );
+        nNear = p->pParent->nNear;
+        pPhrase = (
+            p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
+        );
+        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
+      }
    }
+
+    sqlite3_free(aTmp);
  }

  return res;
@@ -654,6 +654,7 @@ struct Fts5Token {
 /* Parse a MATCH expression. */
 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig, 
+  int iCol,                       /* Column on LHS of MATCH operator */
  const char *zExpr,
  Fts5Expr **ppNew, 
  char **pzErr
@@ -738,7 +739,7 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
-void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
+void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
@@ -213,6 +213,7 @@ static void fts5ParseFree(void *p){ sqlite3_free(p); }

 int sqlite3Fts5ExprNew(
  Fts5Config *pConfig,            /* FTS5 Configuration */
+  int iCol,
  const char *zExpr,              /* Expression text */
  Fts5Expr **ppNew, 
  char **pzErr
@@ -237,6 +238,18 @@ int sqlite3Fts5ExprNew(
  }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

+  /* If the LHS of the MATCH expression was a user column, apply the
+  ** implicit column-filter.  */
+  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
+    int n = sizeof(Fts5Colset);
+    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
+    if( pColset ){
+      pColset->nCol = 1;
+      pColset->aiCol[0] = iCol;
+      sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
+    }
+  }
+
  assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
@@ -1886,25 +1899,110 @@ Fts5Colset *sqlite3Fts5ParseColset(
  return pRet;
 }

+/*
+** If argument pOrig is NULL, or if (*pRc) is set to anything other than
+** SQLITE_OK when this function is called, NULL is returned. 
+**
+** Otherwise, a copy of (*pOrig) is made into memory obtained from
+** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
+** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
+*/
+static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
+  Fts5Colset *pRet;
+  if( pOrig ){
+    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
+    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
+    if( pRet ){ 
+      memcpy(pRet, pOrig, nByte);
+    }
+  }else{
+    pRet = 0;
+  }
+  return pRet;
+}
+
+/*
+** Remove from colset pColset any columns that are not also in colset pMerge.
+*/
+static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
+  int iIn = 0;          /* Next input in pColset */
+  int iMerge = 0;       /* Next input in pMerge */
+  int iOut = 0;         /* Next output slot in pColset */
+
+  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
+    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
+    if( iDiff==0 ){
+      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
+      iMerge++;
+      iIn++;
+    }else if( iDiff>0 ){
+      iMerge++;
+    }else{
+      iIn++;
+    }
+  }
+  pColset->nCol = iOut;
+}
+
+/*
+** Recursively apply colset pColset to expression node pNode and all of
+** its decendents. If (*ppFree) is not NULL, it contains a spare copy
+** of pColset. This function may use the spare copy and set (*ppFree) to
+** zero, or it may create copies of pColset using fts5CloneColset().
+*/
+static void fts5ParseSetColset(
+  Fts5Parse *pParse, 
+  Fts5ExprNode *pNode, 
+  Fts5Colset *pColset,
+  Fts5Colset **ppFree
+){
+  if( pParse->rc==SQLITE_OK ){
+    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
+         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
+         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
+    );
+    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
+      Fts5ExprNearset *pNear = pNode->pNear;
+      if( pNear->pColset ){
+        fts5MergeColset(pNear->pColset, pColset);
+        if( pNear->pColset->nCol==0 ){
+          pNode->eType = FTS5_EOF;
+          pNode->xNext = 0;
+        }
+      }else if( *ppFree ){
+        pNear->pColset = pColset;
+        *ppFree = 0;
+      }else{
+        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
+      }
+    }else{
+      int i;
+      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
+      for(i=0; i<pNode->nChild; i++){
+        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
+      }
+    }
+  }
+}
+
+/*
+** Apply colset pColset to expression node pExpr and all of its descendents.
+*/
 void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
-  Fts5ExprNearset *pNear, 
+  Fts5ExprNode *pExpr, 
  Fts5Colset *pColset 
 ){
+  Fts5Colset *pFree = pColset;
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
    pParse->zErr = sqlite3_mprintf(
      "fts5: column queries are not supported (detail=none)"
    );
-    sqlite3_free(pColset);
-    return;
-  }
-
-  if( pNear ){
-    pNear->pColset = pColset;
  }else{
-    sqlite3_free(pColset);
+    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
  }
+  sqlite3_free(pFree);
 }

 static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
@@ -2358,7 +2456,7 @@ static void fts5ExprFunction(

  rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
+    rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr);
  }
  if( rc==SQLITE_OK ){
    char *zText;
@@ -2878,7 +2878,8 @@ static void fts5MultiIterNext2(
 ){
  assert( pIter->bSkipEmpty );
  if( p->rc==SQLITE_OK ){
-    do {
+    *pbNewTerm = 0;
+    do{
      int iFirst = pIter->aFirst[1].iFirst;
      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
      int bNewTerm = 0;
@@ -2891,8 +2892,6 @@ static void fts5MultiIterNext2(
        fts5MultiIterAdvanced(p, pIter, iFirst, 1);
        fts5MultiIterSetEof(pIter);
        *pbNewTerm = 1;
-      }else{
-        *pbNewTerm = 0;
      }
      fts5AssertMultiIterSetup(p, pIter);

@@ -3158,23 +3157,23 @@ static int fts5IndexExtractCol(
  return p - (*pa);
 }

-static int fts5IndexExtractColset (
+static void fts5IndexExtractColset(
+  int *pRc,
  Fts5Colset *pColset,            /* Colset to filter on */
  const u8 *pPos, int nPos,       /* Position list */
  Fts5Buffer *pBuf                /* Output buffer */
 ){
-  int rc = SQLITE_OK;
-  int i;
-
-  fts5BufferZero(pBuf);
-  for(i=0; i<pColset->nCol; i++){
-    const u8 *pSub = pPos;
-    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
-    if( nSub ){
-      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
+  if( *pRc==SQLITE_OK ){
+    int i;
+    fts5BufferZero(pBuf);
+    for(i=0; i<pColset->nCol; i++){
+      const u8 *pSub = pPos;
+      int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
+      if( nSub ){
+        fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
+      }
    }
  }
-  return rc;
 }

 /*
@@ -3298,8 +3297,9 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
      pIter->base.pData = a;
    }else{
+      int *pRc = &pIter->pIndex->rc;
      fts5BufferZero(&pIter->poslist);
-      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
+      fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
      pIter->base.pData = pIter->poslist.p;
      pIter->base.nData = pIter->poslist.n;
    }
@@ -3844,9 +3844,6 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
  Fts5PageWriter *pPage = &pWriter->writer;
  i64 iRowid;

-static int nCall = 0;
-nCall++;
-
  assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );

  /* Set the szLeaf header field. */
@@ -506,6 +506,7 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
 static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  Fts5Table *pTab = (Fts5Table*)pVTab;
  Fts5Config *pConfig = pTab->pConfig;
+  const int nCol = pConfig->nCol;
  int idxFlags = 0;               /* Parameter passed through to xFilter() */
  int bHasMatch;
  int iNext;
@@ -531,24 +532,34 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){

  int aColMap[3];
  aColMap[0] = -1;
-  aColMap[1] = pConfig->nCol;
-  aColMap[2] = pConfig->nCol+1;
+  aColMap[1] = nCol;
+  aColMap[2] = nCol+1;

  /* Set idxFlags flags for all WHERE clause terms that will be used. */
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
-    int j;
-    for(j=0; j<ArraySize(aConstraint); j++){
-      struct Constraint *pC = &aConstraint[j];
-      if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
-        if( p->usable ){
+    int iCol = p->iColumn;
+
+    if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
+     || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol)
+    ){
+      /* A MATCH operator or equivalent */
+      if( p->usable ){
+        idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16);
+        aConstraint[0].iConsIndex = i;
+      }else{
+        /* As there exists an unusable MATCH constraint this is an 
+        ** unusable plan. Set a prohibitively high cost. */
+        pInfo->estimatedCost = 1e50;
+        return SQLITE_OK;
+      }
+    }else{
+      int j;
+      for(j=1; j<ArraySize(aConstraint); j++){
+        struct Constraint *pC = &aConstraint[j];
+        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
          pC->iConsIndex = i;
          idxFlags |= pC->fts5op;
-        }else if( j==0 ){
-          /* As there exists an unusable MATCH constraint this is an 
-          ** unusable plan. Set a prohibitively high cost. */
-          pInfo->estimatedCost = 1e50;
-          return SQLITE_OK;
        }
      }
    }
@@ -1123,6 +1134,7 @@ static int fts5FilterMethod(
  sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
+  int iCol;                       /* Column on LHS of MATCH operator */
  char **pzErrmsg = pConfig->pzErrmsg;

  UNUSED_PARAM(zUnused);
@@ -1153,6 +1165,8 @@ static int fts5FilterMethod(
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++];
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++];
  if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++];
+  iCol = (idxNum>>16);
+  assert( iCol>=0 && iCol<=pConfig->nCol );
  assert( iVal==nVal );
  bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
  pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
@@ -1199,7 +1213,7 @@ static int fts5FilterMethod(
        rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
      }else{
        char **pzErr = &pTab->base.zErrMsg;
-        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
+        rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr);
        if( rc==SQLITE_OK ){
          if( bOrderByRank ){
            pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
@@ -40,7 +40,7 @@
 */
 #if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)

-#include <fts5.h>
+#include "fts5.h"
 #include <string.h>
 #include <assert.h>

@@ -89,32 +89,6 @@ input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
 %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
 %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }

-expr(A) ::= expr(X) AND expr(Y). {
-  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
-}
-expr(A) ::= expr(X) OR expr(Y). {
-  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
-}
-expr(A) ::= expr(X) NOT expr(Y). {
-  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
-}
-
-expr(A) ::= LP expr(X) RP. {A = X;}
-expr(A) ::= exprlist(X).   {A = X;}
-
-exprlist(A) ::= cnearset(X). {A = X;}
-exprlist(A) ::= exprlist(X) cnearset(Y). {
-  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
-}
-
-cnearset(A) ::= nearset(X). { 
-  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
-}
-cnearset(A) ::= colset(X) COLON nearset(Y). { 
-  sqlite3Fts5ParseSetColset(pParse, Y, X);
-  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
-}
-
 %type colset {Fts5Colset*}
 %destructor colset { sqlite3_free($$); }
 %type colsetlist {Fts5Colset*}
@@ -138,6 +112,37 @@ colsetlist(A) ::= STRING(X). {
  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
 }

+expr(A) ::= expr(X) AND expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
+}
+expr(A) ::= expr(X) OR expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
+}
+expr(A) ::= expr(X) NOT expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
+}
+
+expr(A) ::= colset(X) COLON LP expr(Y) RP. {
+  sqlite3Fts5ParseSetColset(pParse, Y, X);
+  A = Y;
+}
+expr(A) ::= LP expr(X) RP. {A = X;}
+expr(A) ::= exprlist(X).   {A = X;}
+
+exprlist(A) ::= cnearset(X). {A = X;}
+exprlist(A) ::= exprlist(X) cnearset(Y). {
+  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
+}
+
+cnearset(A) ::= nearset(X). { 
+  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
+}
+cnearset(A) ::= colset(X) COLON nearset(Y). { 
+  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
+  sqlite3Fts5ParseSetColset(pParse, A, X);
+}
+
+
 %type nearset     {Fts5ExprNearset*}
 %type nearphrases {Fts5ExprNearset*}
 %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
@@ -577,6 +577,20 @@ do_execsql_test 21.1 {
  COMMIT;
 }

+do_execsql_test 22.0 {
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
+  BEGIN;
+    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
+    DELETE FROM t9 WHERE rowid = 2;
+    INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
+  COMMIT;
+}
+
+do_execsql_test 22.1 {
+  SELECT rowid FROM t9('a*')
+} {1}
+
 }


@@ -44,13 +44,42 @@ foreach_detail_mode $::testprefix {
    9 "-c : a"  {1 2 4}
    10 "-\"c\" : a"  {1 2 4}
  } {
-  breakpoint
    do_execsql_test 1.$tn {
      SELECT rowid FROM t1($q)
    } $res
  }

+  foreach {tn q res} {
+    0 {{a} : (a AND ":")}     {}
+    1 "{a b c} : (a AND d)"   {2 3}
+    2 "{a b c} : (a AND b:d)" {3}
+    3 "{a b c} : (a AND d:d)" {}
+    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
+    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
+    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
+    7 "{a b c} : (b:a AND c:b)" {2}
+  } {
+    do_execsql_test 2.$tn {
+      SELECT rowid FROM t1($q)
+    } $res
+  }

+  foreach {tn w res} {
+    0 "a MATCH 'a'" {1}
+    1 "b MATCH 'a'" {2}
+    2 "b MATCH '{a b c} : a'" {2}
+    3 "b MATCH 'a OR b'"      {1 2}
+    4 "b MATCH 'a OR a:b'"    {2}
+    5 "b MATCH 'a OR b:b'"    {1 2}
+  } {
+    do_execsql_test 3.$tn "
+      SELECT rowid FROM t1 WHERE $w
+    " $res
+  }
+
+  do_catchsql_test 4.1 {
+    SELECT * FROM t1 WHERE rowid MATCH 'a'
+  } {1 {unable to use function MATCH in the requested context}}
 }


@@ -107,5 +107,29 @@ do_faultsim_test 3.3 -faults oom* -body {
  faultsim_test_result {0 {2 3}}
 }

+#-------------------------------------------------------------------------
+# Test OOM injection with nested colsets.
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
+  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
+  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
+  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
+  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
+}
+do_faultsim_test 4.1 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
+} -test {
+  faultsim_test_result {0 2}
+}
+
+do_faultsim_test 4.2 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
+} -test {
+  faultsim_test_result {0 {2 3}}
+}
+
+
 finish_test

@@ -30,7 +30,7 @@ do_eqp_test 1.1 {
  SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
 } {
  0 0 0 {SCAN TABLE t1} 
-  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
+  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
 }

 do_eqp_test 1.2 {
@@ -43,7 +43,7 @@ do_eqp_test 1.2 {
 do_eqp_test 1.3 {
  SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
+  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
 }

@@ -0,0 +1,58 @@
+/*
+** 2017-04-16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file implements a run-time loadable extension to SQLite that
+** registers a sqlite3_collation_needed() callback to register a fake
+** collating function for any unknown collating sequence.  The fake
+** collating function works like BINARY.
+**
+** This extension can be used to load schemas that contain one or more
+** unknown collating sequences.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+
+static int anyCollFunc(
+  void *NotUsed,
+  int nKey1, const void *pKey1,
+  int nKey2, const void *pKey2
+){
+  int rc, n;
+  n = nKey1<nKey2 ? nKey1 : nKey2;
+  rc = memcmp(pKey1, pKey2, n);
+  if( rc==0 ) rc = nKey1 - nKey2;
+  return rc;
+}
+
+static void anyCollNeeded(
+  void *NotUsed,
+  sqlite3 *db,
+  int eTextRep,
+  const char *zCollName
+){
+  sqlite3_create_collation(db, zCollName, eTextRep, 0, anyCollFunc); 
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_anycollseq_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_collation_needed(db, 0, anyCollNeeded);
+  return rc;
+}
@@ -324,54 +324,86 @@ static void output_formatted(DState *p, const char *zFormat, ...){
 }

 /*
-** Output the given string as a quoted string using SQL quoting conventions.
+** Find a string that is not found anywhere in z[].  Return a pointer
+** to that string.
 **
-** The "\n" and "\r" characters are converted to char(10) and char(13)
-** to prevent them from being transformed by end-of-line translators.
+** Try to use zA and zB first.  If both of those are already found in z[]
+** then make up some string and store it in the buffer zBuf.
 */
-static void output_quoted_string(DState *p, const unsigned char *z){
+static const char *unused_string(
+  const char *z,                    /* Result must not appear anywhere in z */
+  const char *zA, const char *zB,   /* Try these first */
+  char *zBuf                        /* Space to store a generated string */
+){
+  unsigned i = 0;
+  if( strstr(z, zA)==0 ) return zA;
+  if( strstr(z, zB)==0 ) return zB;
+  do{
+    sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++);
+  }while( strstr(z,zBuf)!=0 );
+  return zBuf;
+}
+
+/*
+** Output the given string as a quoted string using SQL quoting conventions.
+** Additionallly , escape the "\n" and "\r" characters so that they do not
+** get corrupted by end-of-line translation facilities in some operating
+** systems.
+*/
+static void output_quoted_escaped_string(DState *p, const char *z){
  int i;
  char c;
-  int inQuote = 0;
-  int bStarted = 0;
-
  for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){}
  if( c==0 ){
-    output_formatted(p, "'%s'", z);
-    return;
-  }
-  while( *z ){
-    for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){}
-    if( c=='\'' ) i++;
-    if( i ){
-      if( !inQuote ){
-        if( bStarted ) p->xCallback("||", p->pArg);
-        p->xCallback("'", p->pArg);
-        inQuote = 1;
+    output_formatted(p,"'%s'",z);
+  }else{
+    const char *zNL = 0;
+    const char *zCR = 0;
+    int nNL = 0;
+    int nCR = 0;
+    char zBuf1[20], zBuf2[20];
+    for(i=0; z[i]; i++){
+      if( z[i]=='\n' ) nNL++;
+      if( z[i]=='\r' ) nCR++;
+    }
+    if( nNL ){
+      p->xCallback("replace(", p->pArg);
+      zNL = unused_string(z, "\\n", "\\012", zBuf1);
+    }
+    if( nCR ){
+      p->xCallback("replace(", p->pArg);
+      zCR = unused_string(z, "\\r", "\\015", zBuf2);
+    }
+    p->xCallback("'", p->pArg);
+    while( *z ){
+      for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){}
+      if( c=='\'' ) i++;
+      if( i ){
+        output_formatted(p, "%.*s", i, z);
+        z += i;
      }
-      output_formatted(p, "%.*s", i, z);
-      z += i;
-      bStarted = 1;
+      if( c=='\'' ){
+        p->xCallback("'", p->pArg);
+        continue;
+      }
+      if( c==0 ){
+        break;
+      }
+      z++;
+      if( c=='\n' ){
+        p->xCallback(zNL, p->pArg);
+        continue;
+      }
+      p->xCallback(zCR, p->pArg);
    }
-    if( c=='\'' ){
-      p->xCallback("'", p->pArg);
-      continue;
+    p->xCallback("'", p->pArg);
+    if( nCR ){
+      output_formatted(p, ",'%s',char(13))", zCR);
    }
-    if( inQuote ){
-      p->xCallback("'", p->pArg);
-      inQuote = 0;
+    if( nNL ){
+      output_formatted(p, ",'%s',char(10))", zNL);
    }
-    if( c==0 ){
-      break;
-    }
-    for(i=0; (c = z[i])=='\r' || c=='\n'; i++){
-      if( bStarted ) p->xCallback("||", p->pArg);
-      output_formatted(p, "char(%d)", c);
-      bStarted = 1;
-    }
-    z += i;
  }
-  if( inQuote ) p->xCallback("'", p->pArg);
 }

 /*
@@ -495,7 +527,8 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
              break;
            }
            case SQLITE_TEXT: {
-              output_quoted_string(p, sqlite3_column_text(pStmt,i));
+              output_quoted_escaped_string(p, 
+                   (const char*)sqlite3_column_text(pStmt,i));
              break;
            }
            case SQLITE_BLOB: {
@@ -90,6 +90,7 @@ static const char jsonIsSpace[] = {
  ** but the definitions need to be repeated for separate compilation. */
  typedef sqlite3_uint64 u64;
  typedef unsigned int u32;
+  typedef unsigned short int u16;
  typedef unsigned char u8;
 #endif

@@ -169,8 +170,18 @@ struct JsonParse {
  u32 *aUp;          /* Index of parent of each node */
  u8 oom;            /* Set to true if out of memory */
  u8 nErr;           /* Number of errors seen */
+  u16 iDepth;        /* Nesting depth */
 };

+/*
+** Maximum nesting depth of JSON for this implementation.
+**
+** This limit is needed to avoid a stack overflow in the recursive
+** descent parser.  A depth of 2000 is far deeper than any sane JSON
+** should go.
+*/
+#define JSON_MAX_DEPTH  2000
+
 /**************************************************************************
 ** Utility routines for dealing with JsonString objects
 **************************************************************************/
@@ -735,8 +746,10 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
    if( iThis<0 ) return -1;
    for(j=i+1;;j++){
      while( safe_isspace(z[j]) ){ j++; }
+      if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1;
      x = jsonParseValue(pParse, j);
      if( x<0 ){
+        pParse->iDepth--;
        if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1;
        return -1;
      }
@@ -749,6 +762,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
      if( z[j]!=':' ) return -1;
      j++;
      x = jsonParseValue(pParse, j);
+      pParse->iDepth--;
      if( x<0 ) return -1;
      j = x;
      while( safe_isspace(z[j]) ){ j++; }
@@ -765,7 +779,9 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
    if( iThis<0 ) return -1;
    for(j=i+1;;j++){
      while( safe_isspace(z[j]) ){ j++; }
+      if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1;
      x = jsonParseValue(pParse, j);
+      pParse->iDepth--;
      if( x<0 ){
        if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1;
        return -1;
@@ -785,7 +801,10 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
    j = i+1;
    for(;;){
      c = z[j];
-      if( c<=0x1f ) return -1;  /* Control characters not allowed in strings */
+      if( (c & ~0x1f)==0 ){
+        /* Control characters are not allowed in strings */
+        return -1;
+      }
      if( c=='\\' ){
        c = z[++j];
        if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f'
@@ -885,6 +904,7 @@ static int jsonParse(
  i = jsonParseValue(pParse, 0);
  if( pParse->oom ) i = -1;
  if( i>0 ){
+    assert( pParse->iDepth==0 );
    while( safe_isspace(zJson[i]) ) i++;
    if( zJson[i] ) i = -1;
  }
@@ -1385,7 +1405,7 @@ static void jsonExtractFunc(
 */
 static JsonNode *jsonMergePatch(
  JsonParse *pParse,   /* The JSON parser that contains the TARGET */
-  int iTarget,         /* Node of the TARGET in pParse */
+  u32 iTarget,         /* Node of the TARGET in pParse */
  JsonNode *pPatch     /* The PATCH */
 ){
  u32 i, j;
@@ -82,10 +82,10 @@ int main(int argc, char **argv){
  int bVacuum = 0;
  int rc;
  sqlite3_int64 nProgress = 0;
-  int nArg = argc-2;
+  int nArgc = argc-2;

  if( argc<3 ) usage(argv[0]);
-  for(i=1; i<nArg; i++){
+  for(i=1; i<nArgc; i++){
    const char *zArg = argv[i];
    int nArg = strlen(zArg);
    if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
@@ -1,8 +1,8 @@
-C Improved\scomments.\s\sFix\sa\sproblem\swhen\san\sindexed\sexpression\sis\sused\sin\san\nORDER\sBY\sclause.
-D 2017-04-11T01:01:27.332
+C Minor\ssize\sand\sperformance\simprovements\sto\sthe\spush-down\soptimization.
+D 2017-04-29T14:56:55.202
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
+F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
 F VERSION 0a0e02e16b44ea735b40118fc844311b2ab0d35b25fbeda5120aee62f973f663
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -70,7 +70,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c f0d5de1bc2155ba7cd7c0c1a751779a3a8857fa34d5c12f3b233a23fa2e79ea2
+F ext/fts3/fts3.c 10fc22119e3d91997eb5820d96ff709ca7c61b6f767e09b360b986b897ad74c6
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a
 F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
@@ -99,26 +99,26 @@ F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c495
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
-F ext/fts5/fts5Int.h 88c1a9a01cd5cd7b9a3c95699d1f61cdb277b2fa087a07d3fc989b49970437d9
+F ext/fts5/fts5Int.h 15e7514b46a845937d7c62e5c69e935091f0dbb72eb61aa4c8bcfbd39fdea158
 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c
+F ext/fts5/fts5_expr.c f2825f714d91bbe62ab5820aee9ad12e0c94205b2a01725eaa9072415ae9ff1c
 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
-F ext/fts5/fts5_index.c a3a9ae1c6f8a3652eb45dd732a71cdc92b48df047af134da38cd0c425649e042
-F ext/fts5/fts5_main.c 24cafdc44c06b9665d73c34b75966f6223e9299cc9fd10184c6bf888e3ff563f
+F ext/fts5/fts5_index.c dc25123df20c60492857de491a194dab4b46ace217b8483bda305d357bf6431d
+F ext/fts5/fts5_main.c 1ba0e7806886c1bc16e20d0dde1c2b535d1aeb98cbbb937c4c3e064af5ac6f03
 F ext/fts5/fts5_storage.c 7750986004f3f0c94619a85ecb5dd6cbef53e5e3853488e8a906c269d4d11db6
 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a
 F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
-F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
+F ext/fts5/fts5_test_tok.c f4c27f2e3200653a458f5316f6aaa863f7532d98fff7534ece3670fb7f2c196b
 F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf
 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
 F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7
-F ext/fts5/fts5parse.y e51b375403421b8b37428a89b095d00597129aae
+F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
-F ext/fts5/test/fts5aa.test b3cb080db4851580705c5e261c9d4308edf030d5cbbc24b5d6668403b73f023b
+F ext/fts5/test/fts5aa.test 6dd1bfa4426a8d77e8c8d447aad02515c85118b082f24053ac89802300077ff1
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
@@ -136,7 +136,7 @@ F ext/fts5/test/fts5aux.test 5dd158a1e7869e27e9762a2a452b189c728d1be3
 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
 F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
 F ext/fts5/test/fts5bigtok.test 017a9397b14e7598883a6328ead4a6539b42d59a
-F ext/fts5/test/fts5colset.test 1cdf56e079316005aabda790059aee86f2222ee4
+F ext/fts5/test/fts5colset.test 97084e7fd3183a3c148ade1d4ad980b7e764cb2711ac77c3176680c3e75ea2b9
 F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07
 F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368
 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
@@ -160,7 +160,7 @@ F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080
 F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85
 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08
 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea
-F ext/fts5/test/fts5faultB.test 7f3bba790fa172073ac314f9b8ed197390b61eca
+F ext/fts5/test/fts5faultB.test 8a7e4c89c663e599e88fa1b9f2b8dcf4b85a338bd250e1a94341773b1e9ebfcc
 F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9
@@ -175,7 +175,7 @@ F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
 F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c
 F ext/fts5/test/fts5optimize.test 9d3ac53bb9cae58cb070d795db86bcb2f9fec105
 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f
-F ext/fts5/test/fts5plan.test 6a55ecbac9890765b0e16f8c421c7e0888cfe436
+F ext/fts5/test/fts5plan.test fb0ee60c9a092b17bb958dd742f01e3731b520d50b5d61b288c49c7acdc286cb
 F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1
 F ext/fts5/test/fts5porter2.test 2e65633d58a1c525d5af0f6c01e5a59155bb3487
 F ext/fts5/test/fts5prefix.test 6ef7e875738412907b17687d25db39a25cbdaba4
@@ -209,16 +209,17 @@ F ext/icu/icu.c 84900472a088a3a172c6c079f58a1d3a1952c332
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234
+F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 531a46cbad789fca0aa9db69a0e6c8ac9e68767d
-F ext/misc/dbdump.c 34174d537318027b159e3341105866c05a4149376ede59988c82280bed144e6d
+F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
-F ext/misc/json1.c 70d49f69ce61e54a83a29e425e704ca3e7e42e6bd9a7cf3c112d0ad995f6560b
+F ext/misc/json1.c bd3bbdefb6024c5e40287d61cdf876587c18a6e60fbe8cd6bcdde10eefd14bb1
 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
@@ -236,7 +237,7 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4
+F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d
 F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702
@@ -344,19 +345,19 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 24ae5472bd0b53b4130ecdda389deb621af721d1fcb50890b878102b00bd10fa
+F src/btree.c a0d9a1c782ff3d22df5d217a4fa7125dd69ad5849caa51c4442c10246ca8ae27
 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc
 F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610
 F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c
-F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae
+F src/date.c cc42a41c7422389860d40419a5e3bce5eaf6e7835c3ba2677751dc653550a5c7
 F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d
 F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c
-F src/expr.c 8fd6b7bc10c2c33e017b0f2715efdc83c01e348be27797022ed9eaa3002ca720
+F src/expr.c f10e35dc50be4c8f82eb99bf5d8530229d1d60957cc3c9473ffe584d0444087c
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
-F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae
+F src/fkey.c db65492ae549c3b548c9ef1f279ce1684f1c473b116e1c56a90878cd5dcf968d
 F src/func.c 9d52522cc8ae7f5cdadfe14594262f1618bc1f86083c4cd6da861b4cf5af6174
 F src/global.c 4a34512d82fc5aa13c802db06bcfff5e1d3de955
 F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd
@@ -390,19 +391,19 @@ F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
-F src/parse.y 48b03113704ee8bd78ee6996d81de7fbee22e105
+F src/parse.y 0513387ce02fea97897d8caef82d45f347818593f24f1bdc48e0c530a8af122d
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 1195a21fe28e223e024f900b2011e80df53793f0356a24caace4188b098540dc
-F src/pragma.c 220474f113cade6a6b5bacd3e3a65eca339acbee804128bc5db13a9cad55fba5
+F src/pragma.c 2362670a9d28b71708aecb2b9b10b3f7be71f4c950961c07e81dc400e3ce6371
 F src/pragma.h 37a1311d0388db480388d7ec09054f7103045eff20d4971f8a433b77f40b9921
 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a
 F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 9228235fdefe6de138ac01c4c68d07fd95d78780ef85112a76e63e260a6f5897
-F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d
+F src/select.c bf8ab605e49717c222136380453cfb7eda564f8e500d5ff6a01341ea59fefe80
+F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1
 F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
@@ -427,7 +428,7 @@ F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96
 F src/test_blob.c f65ac717da2618691cf9dad094e6da0219dcd208
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
 F src/test_config.c edcba290248dc18736dd814c9b95863c6762e0b35753048d8cbe5bf65f7abfbb
-F src/test_delete.c af7eab5702f853fb1c62a5f7665e2234cf1ae17b
+F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58
 F src/test_fs.c e16cbe68d3b107e00a907c20a9a02629870eb69b
@@ -462,30 +463,30 @@ F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
-F src/treeview.c 84d0ac737e1231702679f0289180021e19c5cc186ec413e8dcb704a887c76ec8
+F src/treeview.c 6cf8d7fe9e63fae57dad1bb57f6615e14eac0c527e43d868e805042cae8ed1f7
 F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182
-F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58
+F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6
 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569
-F src/vdbe.c 808fda3d50f544120d27c731449b524b4ec8f8b0f734b228831078f0ba53ecb9
+F src/vdbe.c 1eba07ffa0e4ff5e14ceca17e366c26f58c11d12cc56e55a9a13157a93533b7e
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade
 F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860
 F src/vdbeaux.c 6b3f6ce909e206d4c918988b13b7fa687e92b4471d137e0f2a37edac80ec60be
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
-F src/vdbemem.c 934f9bce97228df9d2ae7f094e1ffcd5ada8fe9c0b13cf5c3978c4b7e64f1087
+F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
-F src/vtab.c 007513c2ef52472fcdea6a741683d50662e82790
+F src/vtab.c 35b9bdc2b41de32a417141d12097bcc4e29a77ed7cdb8f836d1d2305d946b61b
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c 1d14e18f32231fa7969e718e7b60ef749b0065e2a7e1b6b00883b20732d280f1
-F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df
-F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681
-F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c
+F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566
+F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20
+F src/wherecode.c 8ad48867660519e262a401720845dc76934f86f558ec9606335fafcd7a2554f8
+F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -575,6 +576,7 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3
 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0
 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de
 F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61
+F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9
 F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738
 F test/capi3.test f0c66919e43d42e1572a69be039e4527a931b00f
 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4
@@ -658,6 +660,7 @@ F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d
 F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2
 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e
 F test/distinct.test a1783b960ad8c15a77cd9f207be072898db1026c
+F test/distinct2.test faef8a3f27424e2cfbe19b2a40752294de3b5d957e049e3336be53ec0476cb58
 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
 F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05
 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d
@@ -706,7 +709,7 @@ F test/fkey1.test ba64806ff9a04eecab2679caad377ae99a5e94e4
 F test/fkey2.test 155809016fad6b2a1491facf2ac53a551bc57c2c
 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49
 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d
-F test/fkey5.test 5a373303f201ac03c22ba1ef17a733d3f56e611a
+F test/fkey5.test 19a9b73bafd78323c37fc883dba060191541503d6094f8aea67fe94d33118e20
 F test/fkey6.test d078a1e323a740062bed38df32b8a736fd320dc0
 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13
 F test/fkey8.test e5372e32cdb4481f121ec3550703eeb7b4e0762c
@@ -750,7 +753,7 @@ F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
 F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
 F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654
 F test/fts3_common.tcl 99cf6659b87c0f74f55963c2aea03b3a7d66ceb0
-F test/fts3aa.test 6c263a6f8845205ee02550981a94c2e8dc1e7058
+F test/fts3aa.test 39b65c11913d277c91d7426c62cfc1d147d1b4e9a48fecd9e38f60d0b5a5f505
 F test/fts3ab.test 7f6cf260ae80dda064023df8e8e503e9a412b91f
 F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63
 F test/fts3ad.test e40570cb6f74f059129ad48bcef3d7cbc20dda49
@@ -775,6 +778,7 @@ F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c
 F test/fts3conf.test 60317efd562080e198b5bdc9fcd222ce32cf01d7
 F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb
 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
+F test/fts3corrupt3.test 56e0ee83e90b57f5f3644cb7d1b36a067b7b8b19cdf0dedce45e5e13cf752f65
 F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
 F test/fts3d.test d3e9c8fb75135ada06bf3bab4f9666224965d708
 F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963
@@ -787,12 +791,13 @@ F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8
 F test/fts3expr4.test c39a15d676b14fc439d9bf845aa7bddcf4a74dc3
 F test/fts3expr5.test f9abfffbf5e53d48a33e12a1e8f8ba2c551c9b49
-F test/fts3fault.test da49627b280b210ebc6657f76344c7851f10ce66
-F test/fts3fault2.test f953bb3cf903988172270a9a0aafd5a890b0f98f
+F test/fts3fault.test 268e9589f44f49d6694ef39a293f0e0f80c6230fb01cc6f34325412acceb99ae
+F test/fts3fault2.test 536bbe01fe2946ec24b063a5eee813e8fd90354a6ca0b8f941d299c405edd17e
 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
 F test/fts3join.test 34750f3ce1e29b2749eaf0f1be2fa6301c5d50da
 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6
 F test/fts3matchinfo.test ce864e0bd92429df8008f31cf557269ba172482a
+F test/fts3misc.test f481128013b9555babdf3bc04c58ab59d59bebc24b5f780f50342b9ffe05b547
 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905
 F test/fts3offsets.test b85fd382abdc78ebce721d8117bd552dfb75094c
 F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2
@@ -812,7 +817,7 @@ F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01
 F test/fts4growth.test e5390da74619cacc389711bac9349640b32c4f9a
 F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269
 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d
-F test/fts4langid.test 65a7332c9bc257919e259a304aa8a38c41655b9d
+F test/fts4langid.test 89a107d36710dc3f44dab0861a784ffab967bd1f32c8d700b031748b9d2703c6
 F test/fts4lastrowid.test fa5e157955a3121615ef3e16ff5196e96c9e1e64
 F test/fts4merge.test d2b39f6b1bd4a9738a13540e2d044cba11c43d47
 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
@@ -856,7 +861,7 @@ F test/in.test 20c5529986998949908f889c8208b2cd894b2cc9
 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75
 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068
-F test/in5.test 6c006e0bcd7351b69350ef566e65f244023489e9
+F test/in5.test 7ae37fcd4a5e198291c6ab5f31a5bb3d15397efe8b75a6736d7a95a7b8dd9e08
 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822
 F test/incrblob2.test a5ce5ed1d0b01e2ed347245a21170372528af0a5
 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4
@@ -878,6 +883,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb
 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
 F test/indexexpr1.test 038b3befa74e5a75126b6e9dd2ae5df61c1c7cf7
+F test/indexexpr2.test 68ee9dbe83fcf85e50f4d0bd1f742a082496f2ee5153f4be2a1861db84462bf7
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371
@@ -910,7 +916,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307
 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa
 F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json101.test c0897616f32d95431f37fd291cb78742181980ac
+F test/json101.test 2bd3703a8566e39f8efee8b83bb60aac363fa312e69c94b4024114e0fe77e380
 F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1
 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0
 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa
@@ -1027,7 +1033,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test af720e7d139e7e5417341d0f0eef2b911c0b067852138dc2f5b6a451b5725118
+F test/permutations.test 8aaa22a0f428a7e6b8446b97bc7691a273eaeff5dc290fb9129bf79fa9813a6e
 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1037,6 +1043,7 @@ F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
 F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad
 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
+F test/pushdown.test a5d2e5e66cc94cfb0989a5dd23e77427dd6c4313019155ba9051f52dfc5326aa
 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca
 F test/quick.test 1681febc928d686362d50057c642f77a02c62e57
 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26
@@ -1482,7 +1489,7 @@ F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 6afa640edc7810725aec61c3076ac617c4aaf0b7
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
-F tool/addopcodes.tcl 10c889c4a65ec6c5604e4a47306fa77ff57ae189
+F tool/addopcodes.tcl edbd53806bf20e25af2373ad0c091be4385081c1aa1813b916bf093f94ed8380
 F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x
@@ -1491,12 +1498,12 @@ F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5
 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2
 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1
 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439
-F tool/fuzzershell.c dbf6c26eef936ec78cb0707570de3a4308b2507e
+F tool/fuzzershell.c e1d90a03ca790d7c331c2aae08ca46ff435f1ae1faa6cb9cc48f4687c18fdc6e
 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c 5ccba178a8e8a4b21e1c9232944d23973da38ad7
+F tool/lemon.c f4f1045743e12f86b132253a3192ef92c94bfceb7f41ac41b8e3b373aa78474e
 F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
@@ -1527,7 +1534,7 @@ F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
 F tool/showstat4.c b14159aa062f661b394ba37b6b7b94bfb8012ab9
-F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977
+F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl ab7d9bf68062907282a64b3e12ccbfad47193c5a
 F tool/speed-check.sh 9630ba0468b609c52f48309243d4eb6e9c34deda
@@ -1570,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8978465f335925378d3aa99df6190ce6a1ee6b130205ccc493f2399803844760
-R 1b0a9c05391f59848f876fd33c1876bb
+P afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444
+R d3fbd82e46a4092d676255e41f20df5b
 U drh
-Z 21e604ae631dfab7b9082f6214d51689
+Z bf8e798698af6ed2b80cadbc03ebd2b9
@@ -1 +1 @@
-c59eaf2b7cd2596733f349fc0fe979f71fd29bd73152a0c57066e0b69d5b7a4a
+91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d
@@ -8124,9 +8124,6 @@ int sqlite3BtreeInsert(
    ** btreeMoveto() call */
    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){
      loc = 0;
-    }else if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0
-               && pCur->info.nKey==pX->nKey-1 ){
-      loc = -1;
    }else if( loc==0 ){
      rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc);
      if( rc ) return rc;
@@ -423,8 +423,10 @@ static void computeYMD(DateTime *p){
    p->Y = 2000;
    p->M = 1;
    p->D = 1;
+  }else if( !validJulianDay(p->iJD) ){
+    datetimeError(p);
+    return;
  }else{
-    assert( validJulianDay(p->iJD) );
    Z = (int)((p->iJD + 43200000)/86400000);
    A = (int)((Z - 1867216.25)/36524.25);
    A = Z + 1 + A - (A/4);
@@ -58,7 +58,7 @@ char sqlite3ExprAffinity(Expr *pExpr){
    return sqlite3AffinityType(pExpr->u.zToken, 0);
  }
 #endif
-  if( op==TK_AGG_COLUMN || op==TK_COLUMN ){
+  if( (op==TK_AGG_COLUMN || op==TK_COLUMN) && pExpr->pTab ){
    return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn);
  }
  if( op==TK_SELECT_COLUMN ){
@@ -956,7 +956,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){
  z = pExpr->u.zToken;
  assert( z!=0 );
  assert( z[0]!=0 );
-  assert( n==sqlite3Strlen30(z) );
+  assert( n==(u32)sqlite3Strlen30(z) );
  if( z[1]==0 ){
    /* Wildcard of the form "?".  Assign the next variable number */
    assert( z[0]=='?' );
@@ -3889,6 +3889,17 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
      break;
    }

+    case TK_IF_NULL_ROW: {
+      int addrINR;
+      addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable);
+      sqlite3ExprCachePush(pParse);
+      inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
+      sqlite3ExprCachePop(pParse);
+      sqlite3VdbeJumpHere(v, addrINR);
+      sqlite3VdbeChangeP3(v, addrINR, inReg);
+      break;
+    }
+
    /*
    ** Form A:
    **   CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END
@@ -1087,8 +1087,16 @@ u32 sqlite3FkOldmask(
 ** UPDATE statement modifies the rowid fields of the table.
 **
 ** If any foreign key processing will be required, this function returns
-** true. If there is no foreign key related processing, this function 
-** returns false.
+** non-zero. If there is no foreign key related processing, this function 
+** returns zero.
+**
+** For an UPDATE, this function returns 2 if:
+**
+**   * There are any FKs for which pTab is the child and the parent table, or
+**   * the UPDATE modifies one or more parent keys for which the action is
+**     not "NO ACTION" (i.e. is CASCADE, SET DEFAULT or SET NULL).
+**
+** Or, assuming some other foreign key processing is required, 1.
 */
 int sqlite3FkRequired(
  Parse *pParse,                  /* Parse context */
@@ -1096,12 +1104,13 @@ int sqlite3FkRequired(
  int *aChange,                   /* Non-NULL for UPDATE operations */
  int chngRowid                   /* True for UPDATE that affects rowid */
 ){
+  int eRet = 0;
  if( pParse->db->flags&SQLITE_ForeignKeys ){
    if( !aChange ){
      /* A DELETE operation. Foreign key processing is required if the 
      ** table in question is either the child or parent table for any 
      ** foreign key constraint.  */
-      return (sqlite3FkReferences(pTab) || pTab->pFKey);
+      eRet = (sqlite3FkReferences(pTab) || pTab->pFKey);
    }else{
      /* This is an UPDATE. Foreign key processing is only required if the
      ** operation modifies one or more child or parent key columns. */
@@ -1109,16 +1118,22 @@ int sqlite3FkRequired(

      /* Check if any child key columns are being modified. */
      for(p=pTab->pFKey; p; p=p->pNextFrom){
-        if( fkChildIsModified(pTab, p, aChange, chngRowid) ) return 1;
+        if( 0==sqlite3_stricmp(pTab->zName, p->zTo) ) return 2;
+        if( fkChildIsModified(pTab, p, aChange, chngRowid) ){
+          eRet = 1;
+        }
      }

      /* Check if any parent key columns are being modified. */
      for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){
-        if( fkParentIsModified(pTab, p, aChange, chngRowid) ) return 1;
+        if( fkParentIsModified(pTab, p, aChange, chngRowid) ){
+          if( p->aAction[1]!=OE_None ) return 2;
+          eRet = 1;
+        }
      }
    }
  }
-  return 0;
+  return eRet;
 }

 /*
@@ -193,6 +193,23 @@ columnlist ::= columnlist COMMA columnname carglist.
 columnlist ::= columnname carglist.
 columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}

+// The following directive causes tokens ABORT, AFTER, ASC, etc. to
+// fallback to ID if they will not parse as their original value.
+// This obviates the need for the "id" nonterminal.
+//
+%fallback ID
+  ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
+  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
+  IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
+  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
+  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
+%ifdef SQLITE_OMIT_COMPOUND_SELECT
+  EXCEPT INTERSECT UNION
+%endif SQLITE_OMIT_COMPOUND_SELECT
+  REINDEX RENAME CTIME_KW IF
+  .
+%wildcard ANY.
+
 // Define operator precedence early so that this is the first occurrence
 // of the operator tokens in the grammer.  Keeping the operators together
 // causes them to be assigned integer values that are close together,
@@ -222,23 +239,6 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
 //
 %token_class id  ID|INDEXED.

-// The following directive causes tokens ABORT, AFTER, ASC, etc. to
-// fallback to ID if they will not parse as their original value.
-// This obviates the need for the "id" nonterminal.
-//
-%fallback ID
-  ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
-  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
-  IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
-  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
-  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
-%ifdef SQLITE_OMIT_COMPOUND_SELECT
-  EXCEPT INTERSECT UNION
-%endif SQLITE_OMIT_COMPOUND_SELECT
-  REINDEX RENAME CTIME_KW IF
-  .
-%wildcard ANY.
-

 // And "ids" is an identifer-or-string.
 //
@@ -1329,33 +1329,37 @@ void sqlite3Pragma(
          assert( x==0 );
        }
        addrOk = sqlite3VdbeMakeLabel(v);
-        if( pParent && pIdx==0 ){
-          int iKey = pFK->aCol[0].iFrom;
-          assert( iKey>=0 && iKey<pTab->nCol );
-          if( iKey!=pTab->iPKey ){
-            sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow);
-            sqlite3ColumnDefault(v, pTab, iKey, regRow);
-            sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v);
-          }else{
-            sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow);
-          }
-          sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v);
-          sqlite3VdbeGoto(v, addrOk);
-          sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2);
-        }else{
-          for(j=0; j<pFK->nCol; j++){
-            sqlite3ExprCodeGetColumnOfTable(v, pTab, 0,
-                            aiCols ? aiCols[j] : pFK->aCol[j].iFrom, regRow+j);
-            sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v);
-          }
-          if( pParent ){
-            sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey,
-                              sqlite3IndexAffinityStr(db,pIdx), pFK->nCol);
-            sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0);
-            VdbeCoverage(v);
-          }
+
+        /* Generate code to read the child key values into registers
+        ** regRow..regRow+n. If any of the child key values are NULL, this 
+        ** row cannot cause an FK violation. Jump directly to addrOk in 
+        ** this case. */
+        for(j=0; j<pFK->nCol; j++){
+          int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom;
+          sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j);
+          sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v);
+        }
+
+        /* Generate code to query the parent index for a matching parent
+        ** key. If a match is found, jump to addrOk. */
+        if( pIdx ){
+          sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey,
+              sqlite3IndexAffinityStr(db,pIdx), pFK->nCol);
+          sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0);
+          VdbeCoverage(v);
+        }else if( pParent ){
+          int jmp = sqlite3VdbeCurrentAddr(v)+2;
+          sqlite3VdbeAddOp3(v, OP_SeekRowid, i, jmp, regRow); VdbeCoverage(v);
+          sqlite3VdbeGoto(v, addrOk);
+          assert( pFK->nCol==1 );
+        }
+
+        /* Generate code to report an FK violation to the caller. */
+        if( HasRowid(pTab) ){
+          sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1);
+        }else{
+          sqlite3VdbeAddOp2(v, OP_Null, 0, regResult+1);
        }
-        sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1);
        sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1);
        sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4);
        sqlite3VdbeResolveLabel(v, addrOk);
@@ -112,14 +112,13 @@ Select *sqlite3SelectNew(
 ){
  Select *pNew;
  Select standin;
-  sqlite3 *db = pParse->db;
-  pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) );
+  pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) );
  if( pNew==0 ){
-    assert( db->mallocFailed );
+    assert( pParse->db->mallocFailed );
    pNew = &standin;
  }
  if( pEList==0 ){
-    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0));
+    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(pParse->db,TK_ASTERISK,0));
  }
  pNew->pEList = pEList;
  pNew->op = TK_SELECT;
@@ -132,7 +131,7 @@ Select *sqlite3SelectNew(
  pNew->addrOpenEphm[0] = -1;
  pNew->addrOpenEphm[1] = -1;
  pNew->nSelectRow = 0;
-  if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc));
+  if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc));
  pNew->pSrc = pSrc;
  pNew->pWhere = pWhere;
  pNew->pGroupBy = pGroupBy;
@@ -143,9 +142,9 @@ Select *sqlite3SelectNew(
  pNew->pLimit = pLimit;
  pNew->pOffset = pOffset;
  pNew->pWith = 0;
-  assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 );
-  if( db->mallocFailed ) {
-    clearSelect(db, pNew, pNew!=&standin);
+  assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || pParse->db->mallocFailed!=0 );
+  if( pParse->db->mallocFailed ) {
+    clearSelect(pParse->db, pNew, pNew!=&standin);
    pNew = 0;
  }else{
    assert( pNew->pSrc!=0 || pParse->nErr>0 );
@@ -1691,7 +1690,7 @@ int sqlite3ColumnsFromExprList(
        pColExpr = pColExpr->pRight;
        assert( pColExpr!=0 );
      }
-      if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){
+      if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){
        /* For columns use the column name name */
        int iCol = pColExpr->iColumn;
        pTab = pColExpr->pTab;
@@ -3145,9 +3144,24 @@ static int multiSelectOrderBy(
 #endif

 #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
+
+/* An instance of the SubstContext object describes an substitution edit
+** to be performed on a parse tree.
+**
+** All references to columns in table iTable are to be replaced by corresponding
+** expressions in pEList.
+*/
+typedef struct SubstContext {
+  Parse *pParse;            /* The parsing context */
+  int iTable;               /* Replace references to this table */
+  int iNewTable;            /* New table number */
+  int isLeftJoin;           /* Add TK_IF_NULL_ROW opcodes on each replacement */
+  ExprList *pEList;         /* Replacement expressions */
+} SubstContext;
+
 /* Forward Declarations */
-static void substExprList(Parse*, ExprList*, int, ExprList*);
-static void substSelect(Parse*, Select *, int, ExprList*, int);
+static void substExprList(SubstContext*, ExprList*);
+static void substSelect(SubstContext*, Select*, int);

 /*
 ** Scan through the expression pExpr.  Replace every reference to
@@ -3163,24 +3177,33 @@ static void substSelect(Parse*, Select *, int, ExprList*, int);
 ** of the subquery rather the result set of the subquery.
 */
 static Expr *substExpr(
-  Parse *pParse,      /* Report errors here */
-  Expr *pExpr,        /* Expr in which substitution occurs */
-  int iTable,         /* Table to be substituted */
-  ExprList *pEList    /* Substitute expressions */
+  SubstContext *pSubst,  /* Description of the substitution */
+  Expr *pExpr            /* Expr in which substitution occurs */
 ){
-  sqlite3 *db = pParse->db;
  if( pExpr==0 ) return 0;
-  if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){
+  if( ExprHasProperty(pExpr, EP_FromJoin) && pExpr->iRightJoinTable==pSubst->iTable ){
+    pExpr->iRightJoinTable = pSubst->iNewTable;
+  }
+  if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable ){
    if( pExpr->iColumn<0 ){
      pExpr->op = TK_NULL;
    }else{
      Expr *pNew;
-      Expr *pCopy = pEList->a[pExpr->iColumn].pExpr;
-      assert( pEList!=0 && pExpr->iColumn<pEList->nExpr );
+      Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr;
+      Expr ifNullRow;
+      assert( pSubst->pEList!=0 && pExpr->iColumn<pSubst->pEList->nExpr );
      assert( pExpr->pLeft==0 && pExpr->pRight==0 );
      if( sqlite3ExprIsVector(pCopy) ){
-        sqlite3VectorErrorMsg(pParse, pCopy);
+        sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
      }else{
+        sqlite3 *db = pSubst->pParse->db;
+        if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
+          memset(&ifNullRow, 0, sizeof(ifNullRow));
+          ifNullRow.op = TK_IF_NULL_ROW;
+          ifNullRow.pLeft = pCopy;
+          ifNullRow.iTable = pSubst->iNewTable;
+          pCopy = &ifNullRow;
+        }
        pNew = sqlite3ExprDup(db, pCopy, 0);
        if( pNew && (pExpr->flags & EP_FromJoin) ){
          pNew->iRightJoinTable = pExpr->iRightJoinTable;
@@ -3191,51 +3214,47 @@ static Expr *substExpr(
      }
    }
  }else{
-    pExpr->pLeft = substExpr(pParse, pExpr->pLeft, iTable, pEList);
-    pExpr->pRight = substExpr(pParse, pExpr->pRight, iTable, pEList);
+    pExpr->pLeft = substExpr(pSubst, pExpr->pLeft);
+    pExpr->pRight = substExpr(pSubst, pExpr->pRight);
    if( ExprHasProperty(pExpr, EP_xIsSelect) ){
-      substSelect(pParse, pExpr->x.pSelect, iTable, pEList, 1);
+      substSelect(pSubst, pExpr->x.pSelect, 1);
    }else{
-      substExprList(pParse, pExpr->x.pList, iTable, pEList);
+      substExprList(pSubst, pExpr->x.pList);
    }
  }
  return pExpr;
 }
 static void substExprList(
-  Parse *pParse,       /* Report errors here */
-  ExprList *pList,     /* List to scan and in which to make substitutes */
-  int iTable,          /* Table to be substituted */
-  ExprList *pEList     /* Substitute values */
+  SubstContext *pSubst, /* Description of the substitution */
+  ExprList *pList       /* List to scan and in which to make substitutes */
 ){
  int i;
  if( pList==0 ) return;
  for(i=0; i<pList->nExpr; i++){
-    pList->a[i].pExpr = substExpr(pParse, pList->a[i].pExpr, iTable, pEList);
+    pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr);
  }
 }
 static void substSelect(
-  Parse *pParse,       /* Report errors here */
-  Select *p,           /* SELECT statement in which to make substitutions */
-  int iTable,          /* Table to be replaced */
-  ExprList *pEList,    /* Substitute values */
-  int doPrior          /* Do substitutes on p->pPrior too */
+  SubstContext *pSubst, /* Description of the substitution */
+  Select *p,            /* SELECT statement in which to make substitutions */
+  int doPrior           /* Do substitutes on p->pPrior too */
 ){
  SrcList *pSrc;
  struct SrcList_item *pItem;
  int i;
  if( !p ) return;
  do{
-    substExprList(pParse, p->pEList, iTable, pEList);
-    substExprList(pParse, p->pGroupBy, iTable, pEList);
-    substExprList(pParse, p->pOrderBy, iTable, pEList);
-    p->pHaving = substExpr(pParse, p->pHaving, iTable, pEList);
-    p->pWhere = substExpr(pParse, p->pWhere, iTable, pEList);
+    substExprList(pSubst, p->pEList);
+    substExprList(pSubst, p->pGroupBy);
+    substExprList(pSubst, p->pOrderBy);
+    p->pHaving = substExpr(pSubst, p->pHaving);
+    p->pWhere = substExpr(pSubst, p->pWhere);
    pSrc = p->pSrc;
    assert( pSrc!=0 );
    for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
-      substSelect(pParse, pItem->pSelect, iTable, pEList, 1);
+      substSelect(pSubst, pItem->pSelect, 1);
      if( pItem->fg.isTabFunc ){
-        substExprList(pParse, pItem->u1.pFuncArg, iTable, pEList);
+        substExprList(pSubst, pItem->u1.pFuncArg);
      }
    }
  }while( doPrior && (p = p->pPrior)!=0 );
@@ -3278,8 +3297,8 @@ static void substSelect(
 **        FROM-clause subquery that is a candidate for flattening.  (2b is
 **        due to ticket [2f7170d73bf9abf80] from 2015-02-09.)
 **
-**   (3)  The subquery is not the right operand of a left outer join
-**        (Originally ticket #306.  Strengthened by ticket #3300)
+**   (3)  The subquery is not the right operand of a LEFT JOIN
+**        or the subquery is not itself a join.
 **
 **   (4)  The subquery is not DISTINCT.
 **
@@ -3291,7 +3310,7 @@ static void substSelect(
 **        DISTINCT.
 **
 **   (7)  The subquery has a FROM clause.  TODO:  For subqueries without
-**        A FROM clause, consider adding a FROM close with the special
+**        A FROM clause, consider adding a FROM clause with the special
 **        table sqlite_once that consists of a single row containing a
 **        single NULL.
 **
@@ -3397,6 +3416,8 @@ static int flattenSubquery(
  SrcList *pSubSrc;   /* The FROM clause of the subquery */
  ExprList *pList;    /* The result set of the outer query */
  int iParent;        /* VDBE cursor number of the pSub result set temp table */
+  int iNewParent = -1;/* Replacement table for iParent */
+  int isLeftJoin = 0; /* True if pSub is the right side of a LEFT JOIN */    
  int i;              /* Loop counter */
  Expr *pWhere;                    /* The WHERE clause */
  struct SrcList_item *pSubitem;   /* The subquery */
@@ -3423,7 +3444,7 @@ static int flattenSubquery(
      return 0;                                          /* Restriction (2b)  */
    }
  }
-    
+
  pSubSrc = pSub->pSrc;
  assert( pSubSrc );
  /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants,
@@ -3461,10 +3482,9 @@ static int flattenSubquery(
    return 0; /* Restriction (23) */
  }

-  /* OBSOLETE COMMENT 1:
-  ** Restriction 3:  If the subquery is a join, make sure the subquery is 
-  ** not used as the right operand of an outer join.  Examples of why this
-  ** is not allowed:
+  /*
+  ** If the subquery is the right operand of a LEFT JOIN, then the
+  ** subquery may not be a join itself.  Example of why this is not allowed:
  **
  **         t1 LEFT OUTER JOIN (t2 JOIN t3)
  **
@@ -3474,27 +3494,13 @@ static int flattenSubquery(
  **
  ** which is not at all the same thing.
  **
-  ** OBSOLETE COMMENT 2:
-  ** Restriction 12:  If the subquery is the right operand of a left outer
-  ** join, make sure the subquery has no WHERE clause.
-  ** An examples of why this is not allowed:
-  **
-  **         t1 LEFT OUTER JOIN (SELECT * FROM t2 WHERE t2.x>0)
-  **
-  ** If we flatten the above, we would get
-  **
-  **         (t1 LEFT OUTER JOIN t2) WHERE t2.x>0
-  **
-  ** But the t2.x>0 test will always fail on a NULL row of t2, which
-  ** effectively converts the OUTER JOIN into an INNER JOIN.
-  **
-  ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE:
-  ** Ticket #3300 shows that flattening the right term of a LEFT JOIN
-  ** is fraught with danger.  Best to avoid the whole thing.  If the
-  ** subquery is the right term of a LEFT JOIN, then do not flatten.
+  ** See also tickets #306, #350, and #3300.
  */
  if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){
-    return 0;
+    isLeftJoin = 1;
+    if( pSubSrc->nSrc>1 ){
+      return 0; /* Restriction (3) */
+    }
  }

  /* Restriction 17: If the sub-query is a compound SELECT, then it must
@@ -3703,6 +3709,7 @@ static int flattenSubquery(
      sqlite3IdListDelete(db, pSrc->a[i+iFrom].pUsing);
      assert( pSrc->a[i+iFrom].fg.isTabFunc==0 );
      pSrc->a[i+iFrom] = pSubSrc->a[i];
+      iNewParent = pSubSrc->a[i].iCursor;
      memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i]));
    }
    pSrc->a[iFrom].fg.jointype = jointype;
@@ -3748,6 +3755,9 @@ static int flattenSubquery(
      pSub->pOrderBy = 0;
    }
    pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
+    if( isLeftJoin ){
+      setJoinExpr(pWhere, iNewParent);
+    }
    if( subqueryIsAgg ){
      assert( pParent->pHaving==0 );
      pParent->pHaving = pParent->pWhere;
@@ -3761,7 +3771,13 @@ static int flattenSubquery(
      pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
    }
    if( db->mallocFailed==0 ){
-      substSelect(pParse, pParent, iParent, pSub->pEList, 0);
+      SubstContext x;
+      x.pParse = pParse;
+      x.iTable = iParent;
+      x.iNewTable = iNewParent;
+      x.isLeftJoin = isLeftJoin;
+      x.pEList = pSub->pEList;
+      substSelect(&x, pParent, 0);
    }
  
    /* The flattened query is distinct if either the inner or the
@@ -3864,8 +3880,14 @@ static int pushDownWhereTerms(
  if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){
    nChng++;
    while( pSubq ){
+      SubstContext x;
      pNew = sqlite3ExprDup(pParse->db, pWhere, 0);
-      pNew = substExpr(pParse, pNew, iCursor, pSubq->pEList);
+      x.pParse = pParse;
+      x.iTable = iCursor;
+      x.iNewTable = iCursor;
+      x.isLeftJoin = 0;
+      x.pEList = pSubq->pEList;
+      pNew = substExpr(&x, pNew);
      pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew);
      pSubq = pSubq->pPrior;
    }
@@ -427,6 +427,36 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){
 }
 #endif

+/*
+** Output string zUtf to stream pOut as w characters.  If w is negative,
+** then right-justify the text.  W is the width in UTF-8 characters, not
+** in bytes.  This is different from the %*.*s specification in printf
+** since with %*.*s the width is measured in bytes, not characters.
+*/
+static void utf8_width_print(FILE *pOut, int w, const char *zUtf){
+  int i;
+  int n;
+  int aw = w<0 ? -w : w;
+  char zBuf[1000];
+  if( aw>sizeof(zBuf)/3 ) aw = sizeof(zBuf)/3;
+  for(i=n=0; zUtf[i]; i++){
+    if( (zUtf[i]&0xc0)!=0x80 ){
+      n++;
+      if( n==aw ){
+        do{ i++; }while( (zUtf[i]&0xc0)==0x80 );
+        break;
+      }
+    }
+  }
+  if( n>=aw ){
+    utf8_printf(pOut, "%.*s", i, zUtf);
+  }else if( w<0 ){
+    utf8_printf(pOut, "%*s%s", aw-n, "", zUtf);
+  }else{
+    utf8_printf(pOut, "%s%*s", zUtf, aw-n, "");
+  }
+}
+

 /*
 ** Determines if a string is a number of not.
@@ -1878,13 +1908,8 @@ static int shell_callback(
            p->actualWidth[i] = w;
          }
          if( showHdr ){
-            if( w<0 ){
-              utf8_printf(p->out,"%*.*s%s",-w,-w,azCol[i],
-                      i==nArg-1 ? rowSep : "  ");
-            }else{
-              utf8_printf(p->out,"%-*.*s%s",w,w,azCol[i],
-                      i==nArg-1 ? rowSep : "  ");
-            }
+            utf8_width_print(p->out, w, azCol[i]);
+            utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : "  ");
          }
        }
        if( showHdr ){
@@ -1920,15 +1945,8 @@ static int shell_callback(
          }
          p->iIndent++;
        }
-        if( w<0 ){
-          utf8_printf(p->out,"%*.*s%s",-w,-w,
-              azArg[i] ? azArg[i] : p->nullValue,
-              i==nArg-1 ? rowSep : "  ");
-        }else{
-          utf8_printf(p->out,"%-*.*s%s",w,w,
-              azArg[i] ? azArg[i] : p->nullValue,
-              i==nArg-1 ? rowSep : "  ");
-        }
+        utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue);
+        utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : "  ");
      }
      break;
    }
@@ -19,14 +19,12 @@
 **     above.
 */

-#if SQLITE_OS_WIN
-#  include <io.h>
-#  define F_OK 0
-#else
+#ifndef SQLITE_OS_WIN
 #  include <unistd.h>
+#  include <errno.h>
 #endif
 #include <string.h>
-#include <errno.h>
+#include <assert.h>
 #include "sqlite3.h"

 /* The following #defines are copied from test_multiplex.c */
@@ -57,23 +55,37 @@ static void sqlite3Delete83Name(char *z){
 ** set *pbExists to true and unlink it. Or, if the file does not exist,
 ** set *pbExists to false before returning.
 **
-** If an error occurs, the value of errno is returned. Or, if no error
-** occurs, zero is returned.
+** If an error occurs, non-zero is returned. Or, if no error occurs, zero.
 */
-static int sqlite3DeleteUnlinkIfExists(const char *zFile, int *pbExists){
-  int rc;
+static int sqlite3DeleteUnlinkIfExists(
+  sqlite3_vfs *pVfs,
+  const char *zFile, 
+  int *pbExists
+){
+  int rc = SQLITE_ERROR;
+#if SQLITE_OS_WIN
+  if( pVfs ){
+    if( pbExists ) *pbExists = 1;
+    rc = pVfs->xDelete(pVfs, zFile, 0);
+    if( rc==SQLITE_IOERR_DELETE_NOENT ){
+      if( pbExists ) *pbExists = 0;
+      rc = SQLITE_OK;
+    }
+  }
+#else
+  assert( pVfs==0 );
  rc = access(zFile, F_OK);
  if( rc ){
    if( errno==ENOENT ){ 
      if( pbExists ) *pbExists = 0;
-      return 0; 
+      rc = SQLITE_OK; 
    }
-    return errno;
+  }else{
+    if( pbExists ) *pbExists = 1;
+    rc = unlink(zFile);
  }
-  if( pbExists ) *pbExists = 1;
-  rc = unlink(zFile);
-  if( rc ) return errno;
-  return 0;
+#endif
+  return rc;
 }

 /*
@@ -103,6 +115,12 @@ SQLITE_API int sqlite3_delete_database(
    { "%s-wal%03d",     SQLITE_MULTIPLEX_WAL_8_3_OFFSET, 1 },
  };

+#ifdef SQLITE_OS_WIN
+  sqlite3_vfs *pVfs = sqlite3_vfs_find("win32");
+#else
+  sqlite3_vfs *pVfs = 0;
+#endif
+
  /* Allocate a buffer large enough for any of the files that need to be
  ** deleted.  */
  nBuf = (int)strlen(zFile) + 100;
@@ -113,10 +131,10 @@ SQLITE_API int sqlite3_delete_database(
  ** journal, wal and shm files.  */
  for(i=0; rc==0 && i<sizeof(azFmt)/sizeof(azFmt[0]); i++){
    sqlite3_snprintf(nBuf, zBuf, azFmt[i], zFile);
-    rc = sqlite3DeleteUnlinkIfExists(zBuf, 0);
+    rc = sqlite3DeleteUnlinkIfExists(pVfs, zBuf, 0);
    if( rc==0 && i!=0 ){
      sqlite3Delete83Name(zBuf);
-      rc = sqlite3DeleteUnlinkIfExists(zBuf, 0);
+      rc = sqlite3DeleteUnlinkIfExists(pVfs, zBuf, 0);
    }
  }

@@ -128,7 +146,7 @@ SQLITE_API int sqlite3_delete_database(
      int bExists;
      sqlite3_snprintf(nBuf, zBuf, p->zFmt, zFile, iChunk+p->iOffset);
      if( p->b83 ) sqlite3Delete83Name(zBuf);
-      rc = sqlite3DeleteUnlinkIfExists(zBuf, &bExists);
+      rc = sqlite3DeleteUnlinkIfExists(pVfs, zBuf, &bExists);
      if( bExists==0 || rc!=0 ) break;
    }
  }
@@ -238,7 +238,7 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
 void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
  const char *zBinOp = 0;   /* Binary operator */
  const char *zUniOp = 0;   /* Unary operator */
-  char zFlgs[30];
+  char zFlgs[60];
  pView = sqlite3TreeViewPush(pView, moreToFollow);
  if( pExpr==0 ){
    sqlite3TreeViewLine(pView, "nil");
@@ -246,7 +246,12 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
    return;
  }
  if( pExpr->flags ){
-    sqlite3_snprintf(sizeof(zFlgs),zFlgs,"  flags=0x%x",pExpr->flags);
+    if( ExprHasProperty(pExpr, EP_FromJoin) ){
+      sqlite3_snprintf(sizeof(zFlgs),zFlgs,"  flags=0x%x iRJT=%d",
+                       pExpr->flags, pExpr->iRightJoinTable);
+    }else{
+      sqlite3_snprintf(sizeof(zFlgs),zFlgs,"  flags=0x%x",pExpr->flags);
+    }
  }else{
    zFlgs[0] = 0;
  }
@@ -465,6 +470,11 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
      sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0);
      break;
    }
+    case TK_IF_NULL_ROW: {
+      sqlite3TreeViewLine(pView, "IF-NULL-ROW %d", pExpr->iTable);
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
+      break;
+    }
    default: {
      sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
      break;
@@ -285,7 +285,7 @@ void sqlite3Update(
  */
  for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
    int reg;
-    if( chngKey || hasFK || pIdx->pPartIdxWhere || pIdx==pPk ){
+    if( chngKey || hasFK>1 || pIdx->pPartIdxWhere || pIdx==pPk ){
      reg = ++pParse->nMem;
      pParse->nMem += pIdx->nColumn;
    }else{
@@ -640,7 +640,7 @@ void sqlite3Update(
    assert( regNew==regNewRowid+1 );
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
    sqlite3VdbeAddOp3(v, OP_Delete, iDataCur,
-        OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP),
+        OPFLAG_ISUPDATE | ((hasFK>1 || chngKey) ? 0 : OPFLAG_ISNOOP),
        regNewRowid
    );
    if( eOnePass==ONEPASS_MULTI ){
@@ -651,7 +651,7 @@ void sqlite3Update(
      sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
    }
 #else
-    if( hasFK || chngKey ){
+    if( hasFK>1 || chngKey ){
      sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0);
    }
 #endif
@@ -486,6 +486,7 @@ static void registerTrace(int iReg, Mem *p){
  printf("REG[%d] = ", iReg);
  memTracePrint(p);
  printf("\n");
+  sqlite3VdbeCheckMemInvariants(p);
 }
 #endif

@@ -1151,7 +1152,7 @@ case OP_Null: {           /* out2 */
 case OP_SoftNull: {
  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
  pOut = &aMem[pOp->p1];
-  pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined;
+  pOut->flags = (pOut->flags&~(MEM_Undefined|MEM_AffMask))|MEM_Null;
  break;
 }

@@ -1494,7 +1495,6 @@ case OP_Remainder: {           /* same as TK_REM, in1, in2, out3 */
  type2 = numericType(pIn2);
  pOut = &aMem[pOp->p3];
  flags = pIn1->flags | pIn2->flags;
-  if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null;
  if( (type1 & type2 & MEM_Int)!=0 ){
    iA = pIn1->u.i;
    iB = pIn2->u.i;
@@ -1518,6 +1518,8 @@ case OP_Remainder: {           /* same as TK_REM, in1, in2, out3 */
    }
    pOut->u.i = iB;
    MemSetTypeFlag(pOut, MEM_Int);
+  }else if( (flags & MEM_Null)!=0 ){
+    goto arithmetic_result_is_null;
  }else{
    bIntint = 0;
 fp_math:
@@ -2429,6 +2431,23 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
  break;
 }

+/* Opcode: IfNullRow P1 P2 P3 * *
+** Synopsis: if P1.nullRow then r[P3]=NULL, goto P2
+**
+** Check the cursor P1 to see if it is currently pointing at a NULL row.
+** If it is, then set register P3 to NULL and jump immediately to P2.
+** If P1 is not on a NULL row, then fall through without making any
+** changes.
+*/
+case OP_IfNullRow: {         /* jump */
+  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+  if( p->apCsr[pOp->p1]->nullRow ){
+    sqlite3VdbeMemSetNull(aMem + pOp->p3);
+    goto jump_to_p2;
+  }
+  break;
+}
+
 /* Opcode: Column P1 P2 P3 P4 P5
 ** Synopsis: r[P3]=PX
 **
@@ -40,6 +40,10 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){
  /* Cannot be both MEM_Int and MEM_Real at the same time */
  assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) );

+  /* Cannot be both MEM_Null and some other type */
+  assert( (p->flags & MEM_Null)==0 ||
+          (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob))==0 );
+
  /* The szMalloc field holds the correct memory allocation size */
  assert( p->szMalloc==0
       || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) );
@@ -142,7 +146,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){
    pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc);
  }

-  if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){
+  if( bPreserve && pMem->z && ALWAYS(pMem->z!=pMem->zMalloc) ){
    memcpy(pMem->zMalloc, pMem->z, pMem->n);
  }
  if( (pMem->flags&MEM_Dyn)!=0 ){
@@ -1053,7 +1053,7 @@ FuncDef *sqlite3VtabOverloadFunction(
  if( NEVER(pExpr==0) ) return pDef;
  if( pExpr->op!=TK_COLUMN ) return pDef;
  pTab = pExpr->pTab;
-  if( NEVER(pTab==0) ) return pDef;
+  if( pTab==0 ) return pDef;
  if( !IsVirtual(pTab) ) return pDef;
  pVtab = sqlite3GetVTable(db, pTab)->pVtab;
  assert( pVtab!=0 );
@@ -3437,7 +3437,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
 }

 /*
-** Examine a WherePath (with the addition of the extra WhereLoop of the 5th
+** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
 ** parameters) to see if it outputs rows in the requested ORDER BY
 ** (or GROUP BY) without requiring a separate sort operation.  Return N:
 ** 
@@ -3532,6 +3532,8 @@ static i8 wherePathSatisfiesOrderBy(
    if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
      if( pLoop->u.vtab.isOrdered ) obSat = obDone;
      break;
+    }else{
+      pLoop->u.btree.nIdxCol = 0;
    }
    iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;

@@ -3677,6 +3679,7 @@ static i8 wherePathSatisfiesOrderBy(
            if( !pColl ) pColl = db->pDfltColl;
            if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
          }
+          pLoop->u.btree.nIdxCol = j+1;
          isMatch = 1;
          break;
        }
@@ -4758,6 +4761,7 @@ WhereInfo *sqlite3WhereBegin(
        if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
         && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
         && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
+         && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
        ){
          sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */
        }
@@ -4846,14 +4850,43 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
    int addr;
    pLevel = &pWInfo->a[i];
    pLoop = pLevel->pWLoop;
-    sqlite3VdbeResolveLabel(v, pLevel->addrCont);
    if( pLevel->op!=OP_Noop ){
+#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
+      int addrSeek = 0;
+      Index *pIdx;
+      int n;
+      if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
+       && (pLoop->wsFlags & WHERE_INDEXED)!=0
+       && (pIdx = pLoop->u.btree.pIndex)->hasStat1
+       && (n = pLoop->u.btree.nIdxCol)>0
+       && pIdx->aiRowLogEst[n]>=36
+      ){
+        int r1 = pParse->nMem+1;
+        int j, op;
+        for(j=0; j<n; j++){
+          sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
+        }
+        pParse->nMem += n+1;
+        op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
+        addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
+        VdbeCoverageIf(v, op==OP_SeekLT);
+        VdbeCoverageIf(v, op==OP_SeekGT);
+        sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
+      }
+#endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
+      /* The common case: Advance to the next row */
+      sqlite3VdbeResolveLabel(v, pLevel->addrCont);
      sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
      sqlite3VdbeChangeP5(v, pLevel->p5);
      VdbeCoverage(v);
      VdbeCoverageIf(v, pLevel->op==OP_Next);
      VdbeCoverageIf(v, pLevel->op==OP_Prev);
      VdbeCoverageIf(v, pLevel->op==OP_VNext);
+#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
+      if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
+#endif
+    }else{
+      sqlite3VdbeResolveLabel(v, pLevel->addrCont);
    }
    if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){
      struct InLoop *pIn;
@@ -4976,6 +5009,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
        }else if( pOp->opcode==OP_Rowid ){
          pOp->p1 = pLevel->iIdxCur;
          pOp->opcode = OP_IdxRowid;
+        }else if( pOp->opcode==OP_IfNullRow ){
+          pOp->p1 = pLevel->iIdxCur;
        }
      }
    }
@@ -124,6 +124,7 @@ struct WhereLoop {
      u16 nEq;               /* Number of equality constraints */
      u16 nBtm;              /* Size of BTM vector */
      u16 nTop;              /* Size of TOP vector */
+      u16 nIdxCol;           /* Index column used for ORDER BY */
      Index *pIndex;         /* Index used, or NULL */
    } btree;
    struct {               /* Information for virtual tables */
@@ -1129,6 +1129,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
  int addrCont;                   /* Jump here to continue with next cycle */
  int iRowidReg = 0;        /* Rowid is stored in this register, if not zero */
  int iReleaseReg = 0;      /* Temp register to free before returning */
+  Index *pIdx = 0;          /* Index used by loop (if any) */
+  int loopAgain;            /* True if constraint generator loop should repeat */

  pParse = pWInfo->pParse;
  v = pParse->pVdbe;
@@ -1454,7 +1456,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
    int endEq;                   /* True if range end uses ==, >= or <= */
    int start_constraints;       /* Start of range is constrained */
    int nConstraint;             /* Number of constraint terms */
-    Index *pIdx;                 /* The index we will be using */
    int iIdxCur;                 /* The VDBE cursor for the index */
    int nExtraReg = 0;           /* Number of extra registers needed */
    int op;                      /* Instruction opcode */
@@ -1705,6 +1706,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
    }else{
      assert( pLevel->p5==0 );
    }
+    if( omitTable ) pIdx = 0;
  }else

 #ifndef SQLITE_OMIT_OR_OPTIMIZATION
@@ -2022,43 +2024,56 @@ Bitmask sqlite3WhereCodeOneLoopStart(

  /* Insert code to test every subexpression that can be completely
  ** computed using the current set of tables.
+  **
+  ** This loop may run either once (pIdx==0) or twice (pIdx!=0). If
+  ** it is run twice, then the first iteration codes those sub-expressions
+  ** that can be computed using columns from pIdx only (without seeking
+  ** the main table cursor). 
  */
-  for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){
-    Expr *pE;
-    int skipLikeAddr = 0;
-    testcase( pTerm->wtFlags & TERM_VIRTUAL );
-    testcase( pTerm->wtFlags & TERM_CODED );
-    if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
-    if( (pTerm->prereqAll & pLevel->notReady)!=0 ){
-      testcase( pWInfo->untestedTerms==0
-               && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
-      pWInfo->untestedTerms = 1;
-      continue;
-    }
-    pE = pTerm->pExpr;
-    assert( pE!=0 );
-    if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){
-      continue;
-    }
-    if( pTerm->wtFlags & TERM_LIKECOND ){
-      /* If the TERM_LIKECOND flag is set, that means that the range search
-      ** is sufficient to guarantee that the LIKE operator is true, so we
-      ** can skip the call to the like(A,B) function.  But this only works
-      ** for strings.  So do not skip the call to the function on the pass
-      ** that compares BLOBs. */
+  do{
+    loopAgain = 0;
+    for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){
+      Expr *pE;
+      int skipLikeAddr = 0;
+      testcase( pTerm->wtFlags & TERM_VIRTUAL );
+      testcase( pTerm->wtFlags & TERM_CODED );
+      if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
+      if( (pTerm->prereqAll & pLevel->notReady)!=0 ){
+        testcase( pWInfo->untestedTerms==0
+            && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
+        pWInfo->untestedTerms = 1;
+        continue;
+      }
+      pE = pTerm->pExpr;
+      assert( pE!=0 );
+      if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){
+        continue;
+      }
+      if( pIdx && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){
+        loopAgain = 1;
+        continue;
+      }
+      if( pTerm->wtFlags & TERM_LIKECOND ){
+        /* If the TERM_LIKECOND flag is set, that means that the range search
+        ** is sufficient to guarantee that the LIKE operator is true, so we
+        ** can skip the call to the like(A,B) function.  But this only works
+        ** for strings.  So do not skip the call to the function on the pass
+        ** that compares BLOBs. */
 #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS
-      continue;
+        continue;
 #else
-      u32 x = pLevel->iLikeRepCntr;
-      assert( x>0 );
-      skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1));
-      VdbeCoverage(v);
+        u32 x = pLevel->iLikeRepCntr;
+        assert( x>0 );
+        skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If, (int)(x>>1));
+        VdbeCoverage(v);
 #endif
+      }
+      sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL);
+      if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr);
+      pTerm->wtFlags |= TERM_CODED;
    }
-    sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL);
-    if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr);
-    pTerm->wtFlags |= TERM_CODED;
-  }
+    pIdx = 0;
+  }while( loopAgain );

  /* Insert code to test for implied constraints based on transitivity
  ** of the "==" operator.
@@ -830,8 +830,8 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
 ** Expression pExpr is one operand of a comparison operator that might
 ** be useful for indexing.  This routine checks to see if pExpr appears
 ** in any index.  Return TRUE (1) if pExpr is an indexed term and return
-** FALSE (0) if not.  If TRUE is returned, also set *piCur to the cursor
-** number of the table that is indexed and *piColumn to the column number
+** FALSE (0) if not.  If TRUE is returned, also set aiCurCol[0] to the cursor
+** number of the table that is indexed and aiCurCol[1] to the column number
 ** of the column that is indexed, or XN_EXPR (-2) if an expression is being
 ** indexed.
 **
@@ -839,18 +839,37 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
 ** true even if that particular column is not indexed, because the column
 ** might be added to an automatic index later.
 */
-static int exprMightBeIndexed(
+static SQLITE_NOINLINE int exprMightBeIndexed2(
  SrcList *pFrom,        /* The FROM clause */
-  int op,                /* The specific comparison operator */
  Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
-  Expr *pExpr,           /* An operand of a comparison operator */
-  int *piCur,            /* Write the referenced table cursor number here */
-  int *piColumn          /* Write the referenced table column number here */
+  int *aiCurCol,         /* Write the referenced table cursor and column here */
+  Expr *pExpr            /* An operand of a comparison operator */
 ){
  Index *pIdx;
  int i;
  int iCur;
-
+  for(i=0; mPrereq>1; i++, mPrereq>>=1){}
+  iCur = pFrom->a[i].iCursor;
+  for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+    if( pIdx->aColExpr==0 ) continue;
+    for(i=0; i<pIdx->nKeyCol; i++){
+      if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
+      if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
+        aiCurCol[0] = iCur;
+        aiCurCol[1] = XN_EXPR;
+        return 1;
+      }
+    }
+  }
+  return 0;
+}
+static int exprMightBeIndexed(
+  SrcList *pFrom,        /* The FROM clause */
+  Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
+  int *aiCurCol,         /* Write the referenced table cursor & column here */
+  Expr *pExpr,           /* An operand of a comparison operator */
+  int op                 /* The specific comparison operator */
+){
  /* If this expression is a vector to the left or right of a 
  ** inequality constraint (>, <, >= or <=), perform the processing 
  ** on the first element of the vector.  */
@@ -862,26 +881,13 @@ static int exprMightBeIndexed(
  }

  if( pExpr->op==TK_COLUMN ){
-    *piCur = pExpr->iTable;
-    *piColumn = pExpr->iColumn;
+    aiCurCol[0] = pExpr->iTable;
+    aiCurCol[1] = pExpr->iColumn;
    return 1;
  }
  if( mPrereq==0 ) return 0;                 /* No table references */
  if( (mPrereq&(mPrereq-1))!=0 ) return 0;   /* Refs more than one table */
-  for(i=0; mPrereq>1; i++, mPrereq>>=1){}
-  iCur = pFrom->a[i].iCursor;
-  for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
-    if( pIdx->aColExpr==0 ) continue;
-    for(i=0; i<pIdx->nKeyCol; i++){
-      if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
-      if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
-        *piCur = iCur;
-        *piColumn = XN_EXPR;
-        return 1;
-      }
-    }
-  }
-  return 0;
+  return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr);
 }

 /*
@@ -961,7 +967,7 @@ static void exprAnalyze(
  pTerm->iParent = -1;
  pTerm->eOperator = 0;
  if( allowedOp(op) ){
-    int iCur, iColumn;
+    int aiCurCol[2];
    Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft);
    Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight);
    u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV;
@@ -972,14 +978,14 @@ static void exprAnalyze(
      pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr;
    }

-    if( exprMightBeIndexed(pSrc, op, prereqLeft, pLeft, &iCur, &iColumn) ){
-      pTerm->leftCursor = iCur;
-      pTerm->u.leftColumn = iColumn;
+    if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){
+      pTerm->leftCursor = aiCurCol[0];
+      pTerm->u.leftColumn = aiCurCol[1];
      pTerm->eOperator = operatorMask(op) & opMask;
    }
    if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
    if( pRight 
-     && exprMightBeIndexed(pSrc, op, pTerm->prereqRight, pRight, &iCur,&iColumn)
+     && exprMightBeIndexed(pSrc, pTerm->prereqRight, aiCurCol, pRight, op)
    ){
      WhereTerm *pNew;
      Expr *pDup;
@@ -1009,8 +1015,8 @@ static void exprAnalyze(
        pNew = pTerm;
      }
      exprCommute(pParse, pDup);
-      pNew->leftCursor = iCur;
-      pNew->u.leftColumn = iColumn;
+      pNew->leftCursor = aiCurCol[0];
+      pNew->u.leftColumn = aiCurCol[1];
      testcase( (prereqLeft | extraRight) != prereqLeft );
      pNew->prereqRight = prereqLeft | extraRight;
      pNew->prereqAll = prereqAll;
@@ -0,0 +1,77 @@
+# 2017 April 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix cachespill
+
+ifcapable !pager_pragmas {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Test that "PRAGMA cache_spill = 0" completely disables cache spilling.
+#
+do_execsql_test 1.1 {
+  PRAGMA auto_vacuum = 0;
+  PRAGMA page_size = 1024;
+  PRAGMA cache_size = 100;
+  CREATE TABLE t1(a);
+}
+
+do_test 1.2 {
+  file size test.db
+} {2048}
+
+do_test 1.3 {
+  execsql {
+    BEGIN;
+      WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200
+      ) INSERT INTO t1 SELECT randomblob(900) FROM s;
+  }
+  expr {[file size test.db] > 50000}
+} {1}
+
+do_test 1.4 {
+  execsql ROLLBACK
+  file size test.db
+} {2048}
+
+do_test 1.5 {
+  execsql {
+    PRAGMA cache_spill = 0;
+    BEGIN;
+      WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200
+      ) INSERT INTO t1 SELECT randomblob(900) FROM s;
+  }
+  file size test.db
+} {2048}
+
+do_test 1.5 {
+  execsql {
+    ROLLBACK;
+    PRAGMA cache_spill = 1;
+    BEGIN;
+      WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200
+      ) INSERT INTO t1 SELECT randomblob(900) FROM s;
+  }
+  expr {[file size test.db] > 50000}
+} {1}
+
+do_execsql_test 1.6 { ROLLBACK }
+
+
+finish_test
@@ -0,0 +1,183 @@
+# 2016-04-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is DISTINCT queries using the skip-ahead 
+# optimization.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set testprefix distinct2
+
+do_execsql_test 100 {
+  CREATE TABLE t1(x INTEGER PRIMARY KEY);
+  INSERT INTO t1 VALUES(0),(1),(2);
+  CREATE TABLE t2 AS
+     SELECT DISTINCT a.x AS aa, b.x AS bb
+      FROM t1 a, t1 b;
+  SELECT *, '|' FROM t2 ORDER BY aa, bb;
+} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |}
+do_execsql_test 110 {
+  DROP TABLE t2;
+  CREATE TABLE t2 AS
+     SELECT DISTINCT a.x AS aa, b.x AS bb
+       FROM t1 a, t1 b
+      WHERE a.x IN t1 AND b.x IN t1;
+  SELECT *, '|' FROM t2 ORDER BY aa, bb;
+} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |}
+do_execsql_test 120 {
+  CREATE TABLE t102 (i0 TEXT UNIQUE NOT NULL);
+  INSERT INTO t102 VALUES ('0'),('1'),('2');
+  DROP TABLE t2;
+  CREATE TABLE t2 AS
+    SELECT DISTINCT * 
+    FROM t102 AS t0 
+    JOIN t102 AS t4 ON (t2.i0 IN t102)
+    NATURAL JOIN t102 AS t3
+    JOIN t102 AS t1 ON (t0.i0 IN t102)
+    JOIN t102 AS t2 ON (t2.i0=+t0.i0 OR (t0.i0<>500 AND t2.i0=t1.i0));
+  SELECT *, '|' FROM t2 ORDER BY 1, 2, 3, 4, 5;
+} {0 0 0 0 | 0 0 1 0 | 0 0 1 1 | 0 0 2 0 | 0 0 2 2 | 0 1 0 0 | 0 1 1 0 | 0 1 1 1 | 0 1 2 0 | 0 1 2 2 | 0 2 0 0 | 0 2 1 0 | 0 2 1 1 | 0 2 2 0 | 0 2 2 2 | 1 0 0 0 | 1 0 0 1 | 1 0 1 1 | 1 0 2 1 | 1 0 2 2 | 1 1 0 0 | 1 1 0 1 | 1 1 1 1 | 1 1 2 1 | 1 1 2 2 | 1 2 0 0 | 1 2 0 1 | 1 2 1 1 | 1 2 2 1 | 1 2 2 2 | 2 0 0 0 | 2 0 0 2 | 2 0 1 1 | 2 0 1 2 | 2 0 2 2 | 2 1 0 0 | 2 1 0 2 | 2 1 1 1 | 2 1 1 2 | 2 1 2 2 | 2 2 0 0 | 2 2 0 2 | 2 2 1 1 | 2 2 1 2 | 2 2 2 2 |}
+
+do_execsql_test 400 {
+  CREATE TABLE t4(a,b,c,d,e,f,g,h,i,j);
+  INSERT INTO t4 VALUES(0,1,2,3,4,5,6,7,8,9);
+  INSERT INTO t4 SELECT * FROM t4;
+  INSERT INTO t4 SELECT * FROM t4;
+  CREATE INDEX t4x ON t4(c,d,e);
+  SELECT DISTINCT a,b,c FROM t4 WHERE a=0 AND b=1;
+} {0 1 2}
+do_execsql_test 410 {
+  SELECT DISTINCT a,b,c,d FROM t4 WHERE a=0 AND b=1;
+} {0 1 2 3}
+do_execsql_test 411 {
+  SELECT DISTINCT d,a,b,c FROM t4 WHERE a=0 AND b=1;
+} {3 0 1 2}
+do_execsql_test 420 {
+  SELECT DISTINCT a,b,c,d,e FROM t4 WHERE a=0 AND b=1;
+} {0 1 2 3 4}
+do_execsql_test 430 {
+  SELECT DISTINCT a,b,c,d,e,f FROM t4 WHERE a=0 AND b=1;
+} {0 1 2 3 4 5}
+
+do_execsql_test 500 {
+  CREATE TABLE t5(a INT, b INT);
+  CREATE UNIQUE INDEX t5x ON t5(a+b);
+  INSERT INTO t5(a,b) VALUES(0,0),(1,0),(1,1),(0,3);
+  CREATE TEMP TABLE out AS SELECT DISTINCT a+b FROM t5;
+  SELECT * FROM out ORDER BY 1;
+} {0 1 2 3}
+
+do_execsql_test 600 {
+  CREATE TABLE t6a(x INTEGER PRIMARY KEY);
+  INSERT INTO t6a VALUES(1);
+  CREATE TABLE t6b(y INTEGER PRIMARY KEY);
+  INSERT INTO t6b VALUES(2),(3);
+  SELECT DISTINCT x, x FROM t6a, t6b;
+} {1 1}
+
+do_execsql_test 700 {
+  CREATE TABLE t7(a, b, c);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE (i+1)<200
+  )
+  INSERT INTO t7 SELECT i/100, i/50, i FROM s;
+}
+do_execsql_test 710 {
+  SELECT DISTINCT a, b FROM t7;
+} {
+  0 0    0 1
+  1 2    1 3
+}
+do_execsql_test 720 {
+  SELECT DISTINCT a, b+1 FROM t7;
+} {
+  0 1    0 2
+  1 3    1 4
+}
+do_execsql_test 730 {
+  CREATE INDEX i7 ON t7(a, b+1);
+  ANALYZE;
+  SELECT DISTINCT a, b+1 FROM t7;
+} {
+  0 1    0 2
+  1 3    1 4
+}
+
+do_execsql_test 800 {
+  CREATE TABLE t8(a, b, c);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE (i+1)<100
+  )
+  INSERT INTO t8 SELECT i/40, i/20, i/40 FROM s;
+}
+
+do_execsql_test 820 {
+  SELECT DISTINCT a, b, c FROM t8;
+} {
+  0 0 0    0 1 0
+  1 2 1    1 3 1
+  2 4 2
+}
+
+do_execsql_test 820 {
+  SELECT DISTINCT a, b, c FROM t8 WHERE b=3;
+} {1 3 1}
+
+do_execsql_test 830 {
+  CREATE INDEX i8 ON t8(a, c);
+  ANALYZE;
+  SELECT DISTINCT a, b, c FROM t8 WHERE b=3;
+} {1 3 1}
+
+do_execsql_test 900 {
+  CREATE TABLE t9(v);
+  INSERT INTO t9 VALUES 
+    ('abcd'), ('Abcd'), ('aBcd'), ('ABcd'), ('abCd'), ('AbCd'), ('aBCd'), 
+    ('ABCd'), ('abcD'), ('AbcD'), ('aBcD'), ('ABcD'), ('abCD'), ('AbCD'), 
+    ('aBCD'), ('ABCD'),
+    ('wxyz'), ('Wxyz'), ('wXyz'), ('WXyz'), ('wxYz'), ('WxYz'), ('wXYz'), 
+    ('WXYz'), ('wxyZ'), ('WxyZ'), ('wXyZ'), ('WXyZ'), ('wxYZ'), ('WxYZ'), 
+    ('wXYZ'), ('WXYZ');
+}
+
+do_execsql_test 910 {
+  SELECT DISTINCT v COLLATE NOCASE, v FROM t9 ORDER BY +v;
+} {
+  ABCD ABCD ABCd ABCd ABcD ABcD ABcd ABcd AbCD
+  AbCD AbCd AbCd AbcD AbcD Abcd Abcd
+  WXYZ WXYZ WXYz WXYz WXyZ WXyZ WXyz WXyz WxYZ
+  WxYZ WxYz WxYz WxyZ WxyZ Wxyz Wxyz
+  aBCD aBCD aBCd aBCd aBcD aBcD aBcd aBcd abCD
+  abCD abCd abCd abcD abcD abcd abcd
+  wXYZ wXYZ wXYz wXYz wXyZ wXyZ wXyz wXyz wxYZ
+  wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz
+}
+
+do_execsql_test 920 {
+  CREATE INDEX i9 ON t9(v COLLATE NOCASE, v);
+  ANALYZE;
+
+  SELECT DISTINCT v COLLATE NOCASE, v FROM t9 ORDER BY +v;
+} {
+  ABCD ABCD ABCd ABCd ABcD ABcD ABcd ABcd AbCD
+  AbCD AbCd AbCd AbcD AbcD Abcd Abcd
+  WXYZ WXYZ WXYz WXYz WXyZ WXyZ WXyz WXyz WxYZ
+  WxYZ WxYz WxYz WxyZ WxyZ Wxyz Wxyz
+  aBCD aBCD aBCd aBCd aBcD aBcD aBcd aBcd abCD
+  abCD abCd abCd abcD abcD abcd abcd
+  wXYZ wXYZ wXYz wXYz wXyZ wXyZ wXyz wXyz wxYZ
+  wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz
+}
+
+
+finish_test
@@ -388,5 +388,40 @@ do_execsql_test 9.4 {
  PRAGMA foreign_key_check(k2);
 } {k2 3 s1 0}

+#-------------------------------------------------------------------------
+# Test using a WITHOUT ROWID table as the child table with an INTEGER 
+# PRIMARY KEY as the parent key.
+#
+reset_db
+do_execsql_test 10.1 {
+  CREATE TABLE p30 (id INTEGER PRIMARY KEY);
+  CREATE TABLE IF NOT EXISTS c30 (
+      line INTEGER, 
+      master REFERENCES p30(id), 
+      PRIMARY KEY(master)
+  ) WITHOUT ROWID;
+
+  INSERT INTO p30 (id) VALUES (1);
+  INSERT INTO c30 (master, line)  VALUES (1, 999);
+}
+do_execsql_test 10.2 {
+  PRAGMA foreign_key_check;
+}
+do_execsql_test 10.3 {
+  INSERT INTO c30 VALUES(45, 45);
+  PRAGMA foreign_key_check;
+} {c30 {} p30 0}
+
+#-------------------------------------------------------------------------
+# Test "foreign key mismatch" errors.
+#
+reset_db
+do_execsql_test 11.0 {
+  CREATE TABLE tt(y);
+  CREATE TABLE c11(x REFERENCES tt(y));
+}
+do_catchsql_test 11.1 {
+  PRAGMA foreign_key_check;
+} {1 {foreign key mismatch - "c11" referencing "tt"}}

 finish_test
@@ -243,5 +243,11 @@ do_execsql_test 8.5 {
  SELECT docid FROM t0 WHERE t0 MATCH '"abc abc"';
 } {}

+do_execsql_test 9.1 {
+  CREATE VIRTUAL TABLE t9 USING fts4(a, "", '---');
+}
+do_execsql_test 9.2 {
+  CREATE VIRTUAL TABLE t10 USING fts3(<, b, c);
+}

 finish_test
@@ -0,0 +1,65 @@
+# 2010 October 27
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Test that the FTS3 extension does not crash when it encounters a
+# corrupt data structure on disk.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
+ifcapable !fts3 { finish_test ; return }
+
+set ::testprefix fts3corrupt3
+
+#-------------------------------------------------------------------------
+# Test that fts3 does not choke on an oversized varint.
+#
+do_execsql_test 1.0 {
+  PRAGMA page_size = 512;
+  CREATE VIRTUAL TABLE t1 USING fts3;
+  BEGIN;
+    INSERT INTO t1 VALUES('one');
+    INSERT INTO t1 VALUES('one');
+    INSERT INTO t1 VALUES('one');
+  COMMIT;
+}
+do_execsql_test 1.1 {
+  SELECT quote(root) from t1_segdir;
+} {X'00036F6E6509010200010200010200'}
+do_execsql_test 1.2 {
+  UPDATE t1_segdir SET root = X'00036F6E650EFFFFFFFFFFFFFFFFFFFFFFFF0200';
+}
+do_catchsql_test 1.3 {
+  SELECT rowid FROM t1 WHERE t1 MATCH 'one'
+} {0 -1}
+
+#-------------------------------------------------------------------------
+# Interior node with the prefix or suffix count of an entry set to a
+# negative value.
+#
+set doc1 [string repeat "x " 600]
+set doc2 [string repeat "y " 600]
+set doc3 [string repeat "z " 600]
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts3;
+  BEGIN;
+    INSERT INTO t2 VALUES($doc1);
+    INSERT INTO t2 VALUES($doc2);
+    INSERT INTO t2 VALUES($doc3);
+  COMMIT;
+}
+do_execsql_test 2.1 {
+  SELECT quote(root) from t2_segdir;
+} {X'0101017900017A'}
+
+
+
+finish_test
@@ -18,8 +18,6 @@ set ::testprefix fts3fault
 # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
 ifcapable !fts3 { finish_test ; return }

-if 0 {
-
 # Test error handling in the sqlite3Fts3Init() function. This is the 
 # function that registers the FTS3 module and various support functions
 # with SQLite.
@@ -160,8 +158,6 @@ do_faultsim_test 7.3 -prep {
 }


-}
-
 proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
@@ -234,4 +230,12 @@ do_faultsim_test 9.1 -prep {
  faultsim_test_result {0 {{0 0 20 39 0 0 64 2}}}
 }

+do_faultsim_test 10.1 -prep {
+  faultsim_delete_and_reopen
+} -body {
+  execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=d) }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
 finish_test
@@ -174,4 +174,76 @@ do_faultsim_test 6.1 -faults oom* -prep {
  faultsim_test_result {0 -1}
 }

+#-------------------------------------------------------------------------
+# Inject faults into a query for an N-byte prefix that uses a prefix=N+1
+# index.
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE t7 USING fts4(x,prefix=2);
+  INSERT INTO t7 VALUES('the quick brown fox');
+  INSERT INTO t7 VALUES('jumped over the');
+  INSERT INTO t7 VALUES('lazy dog');
+}
+do_faultsim_test 7.1 -faults oom* -body {
+  execsql { SELECT docid FROM t7 WHERE t7 MATCH 't*' }
+} -test {
+  faultsim_test_result {0 {1 2}}
+}
+
+#-------------------------------------------------------------------------
+# Inject faults into a opening an existing fts3 table that has been 
+# upgraded to add an %_stat table.
+#
+reset_db
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE t8 USING fts3;
+  INSERT INTO t8 VALUES('the quick brown fox');
+  INSERT INTO t8 VALUES('jumped over the');
+  INSERT INTO t8 VALUES('lazy dog');
+  INSERT INTO t8(t8) VALUES('automerge=8');
+  SELECT name FROM sqlite_master WHERE name LIKE 't8%';
+} {
+  t8 t8_content t8_segments t8_segdir t8_stat
+}
+faultsim_save_and_close
+
+do_faultsim_test 8.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { INSERT INTO t8 VALUES('one two three') }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 8.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { ALTER TABLE t8 RENAME TO t8ii }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+set chunkconfig [fts3_configure_incr_load 1 1]
+do_execsql_test 9.0 {
+  PRAGMA page_size = 512;
+  CREATE VIRTUAL TABLE t9 USING fts3;
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50
+  )
+  INSERT INTO t9 SELECT 'one two three' FROM s;
+}
+
+do_faultsim_test 8.2 -faults io* -body {
+  execsql { SELECT count(*) FROM t9 WHERE t9 MATCH '"one two three"' }
+} -test {
+  faultsim_test_result {0 50}
+}
+
+eval fts3_configure_incr_load $chunkconfig
+
+
 finish_test
+
+
@@ -0,0 +1,228 @@
+# 2017 March 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts3misc
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# A self-join.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts3(a, b);
+  INSERT INTO t1 VALUES('one', 'i');
+  INSERT INTO t1 VALUES('one', 'ii');
+  INSERT INTO t1 VALUES('two', 'i');
+  INSERT INTO t1 VALUES('two', 'ii');
+}
+
+do_execsql_test 1.1 {
+  SELECT a.a, b.b FROM t1 a, t1 b WHERE a.t1 MATCH 'two' AND b.t1 MATCH 'i'
+} {two i two i two i two i}
+
+#-------------------------------------------------------------------------
+# FTS tables with 128 or more columns.
+#
+proc v1 {v} {
+  set vector [list a b c d e f g h]
+  set res [list]
+  for {set i 0} {$i<8} {incr i} {
+    if {$v & (1 << $i)} { lappend res [lindex $vector $i] }
+  }
+  set res
+}
+proc v2 {v} {
+  set vector [list d e f g h i j k]
+  set res [list]
+  for {set i 0} {$i<8} {incr i} {
+    if {$v & (1 << $i)} { lappend res [lindex $vector $i] }
+  }
+  set res
+}
+db func v1 v1
+db func v2 v2
+
+do_test 2.0 {
+  set cols [list]
+  for {set i 0} {$i<200} {incr i} {
+    lappend cols "c$i"
+  }
+  execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
+  execsql {
+    WITH data(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM data WHERE i<200
+    )
+    INSERT INTO t2(c198, c199) SELECT v1(i), v2(i) FROM data;
+  }
+} {}
+do_execsql_test 2.1 {
+  SELECT rowid FROM t2 WHERE t2 MATCH '"a b c"'
+} {
+  7 15 23 31 39 47 55 63 71 79 87 95 103 111 
+  119 127 135 143 151 159 167 175 183 191 199
+}
+do_execsql_test 2.2 {
+  SELECT rowid FROM t2 WHERE t2 MATCH '"g h i"'
+} {
+  56 57 58 59 60 61 62 63 120 121 122 123 124 
+  125 126 127 184 185 186 187 188 189 190 191
+}
+do_execsql_test 2.3 {
+  SELECT rowid FROM t2 WHERE t2 MATCH '"i h"'
+} {
+}
+do_execsql_test 2.4 {
+  SELECT rowid FROM t2 WHERE t2 MATCH '"f e"'
+} {
+}
+do_execsql_test 2.5 {
+  SELECT rowid FROM t2 WHERE t2 MATCH '"e f"'
+} {
+  6 7 14 15 22 23 30 31 38 39 46 47 48 49 50 51 52 53 54 55 56 
+  57 58 59 60 61 62 63 70 71 78 79 86 87 94 95 102 103 110 
+  111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
+  134 135 142 143 150 151 158 159 166 167 174 175 176 177 178 179 180 
+  181 182 183 184 185 186 187 188 189 190 191 198 199
+}
+
+#-------------------------------------------------------------------------
+# Range constraints on the docid using non-integer values.
+#
+do_execsql_test 2.6 {
+  SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN NULL AND 45;
+} {}
+do_execsql_test 2.7 {
+  SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN 11.5 AND 48.2;
+} {
+  14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 
+  29 30 31 34 35 38 39 42 43 46 47 48
+}
+do_execsql_test 2.8 {
+  SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN '11.5' AND '48.2';
+} {
+  14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 
+  29 30 31 34 35 38 39 42 43 46 47 48
+}
+
+#-------------------------------------------------------------------------
+# Phrase query tests. 
+#
+do_execsql_test 3.1.1 {
+  CREATE VIRTUAL TABLE t3 USING fts3;
+  INSERT INTO t3 VALUES('a b c');
+  INSERT INTO t3 VALUES('d e f');
+  INSERT INTO t3 VALUES('a b d');
+  INSERT INTO t3 VALUES('1 2 3 4 5 6 7 8 9 10 11');
+}
+do_execsql_test 3.1.2 {
+  SELECT * FROM t3 WHERE t3 MATCH '"a b x y"' ORDER BY docid DESC
+}
+do_execsql_test 3.1.3 {
+  SELECT * FROM t3 WHERE t3 MATCH '"a b c" OR "a b x y"' ORDER BY docid DESC
+} {{a b c}}
+do_execsql_test 3.1.4 {
+  SELECT * FROM t3 WHERE t3 MATCH '"a* b* x* a*"'
+}
+do_execsql_test 3.1.5 {
+  SELECT rowid FROM t3 WHERE t3 MATCH '"2 3 4 5 6 7 8 9"'
+} {4}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  PRAGMA page_size = 512;
+  CREATE VIRTUAL TABLE t4 USING fts4;
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<8000 )
+  INSERT INTO t4 SELECT 'a b c a b c a b c' FROM s;
+}
+do_execsql_test 4.1 {
+  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
+} {8000}
+do_execsql_test 4.2 {
+  SELECT quote(value) from t4_stat where id=0
+} {X'C03EC0B204C0A608'}
+do_execsql_test 4.3 {
+  UPDATE t4_stat SET value = X'C03EC0B204C0A60800' WHERE id=0;
+}
+do_catchsql_test 4.4 {
+  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
+} {1 {database disk image is malformed}}
+do_execsql_test 4.5 {
+  UPDATE t4_stat SET value = X'00C03EC0B204C0A608' WHERE id=0;
+}
+do_catchsql_test 4.6 {
+  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t5 USING fts4;
+  INSERT INTO t5 VALUES('a x x x x b x x x x c');
+  INSERT INTO t5 VALUES('a x x x x b x x x x c');
+  INSERT INTO t5 VALUES('a x x x x b x x x x c');
+}
+do_execsql_test 5.1 {
+  SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/4 b NEAR/4 c'
+} {1 2 3}
+do_execsql_test 5.2 {
+  SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/3 b NEAR/4 c'
+} {}
+do_execsql_test 5.3 {
+  SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/4 b NEAR/3 c'
+} {}
+do_execsql_test 5.4 {
+  SELECT rowid FROM t5 WHERE t5 MATCH 'y NEAR/4 b NEAR/4 c'
+} {}
+do_execsql_test 5.5 {
+  SELECT rowid FROM t5 WHERE t5 MATCH 'x OR a NEAR/3 b NEAR/3 c'
+} {1 2 3}
+do_execsql_test 5.5 {
+  SELECT rowid FROM t5 WHERE t5 MATCH 'x OR y NEAR/3 b NEAR/3 c'
+} {1 2 3}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE t6 USING fts4;
+
+  BEGIN;
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000)
+    INSERT INTO t6 SELECT 'x x x x x x x x x x x' FROM s;
+
+  INSERT INTO t6 VALUES('x x x x x x x x x x x A');
+  INSERT INTO t6 VALUES('x x x x x x x x x x x B');
+  INSERT INTO t6 VALUES('x x x x x x x x x x x A');
+  INSERT INTO t6 VALUES('x x x x x x x x x x x B');
+
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000)
+    INSERT INTO t6 SELECT 'x x x x x x x x x x x' FROM s;
+  COMMIT;
+}
+breakpoint
+do_execsql_test 6.1 {
+  SELECT rowid FROM t6 WHERE t6 MATCH 'b OR "x a"'
+} {50001 50002 50003 50004}
+
+
+finish_test
@@ -14,7 +14,6 @@

 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set ::testprefix fts4content

 # If SQLITE_ENABLE_FTS3 is defined, omit this file.
 ifcapable !fts3 {
@@ -341,6 +340,13 @@ do_test_query1 3.3.4 {"zero one" OR "one two"} {
  or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
 }

+do_execsql_test 3.4 {
+  CREATE TABLE t8c(a, b);
+  CREATE VIRTUAL TABLE t8 USING fts4(content=t8c, languageid=langid);
+  INSERT INTO t8(docid, a, b) VALUES(-1, 'one two three', 'x y z');
+  SELECT docid FROM t8 WHERE t8 MATCH 'one x' AND langid=0
+} {-1}
+
 #-------------------------------------------------------------------------
 # Test cases 4.*
 #
@@ -237,4 +237,16 @@ do_execsql_test 8.4 {
  SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1)
 } 3

+#-------------------------------------------------------------------------
+# Test that ticket 61fe97454c is fixed.
+#
+do_execsql_test 9.0 {
+  CREATE TABLE t9(a INTEGER PRIMARY KEY);
+  INSERT INTO t9 VALUES (44), (45);
+}
+do_execsql_test 9.1 {
+  SELECT * FROM t9 WHERE a IN (44, 45, 44, 45)
+} {44 45}
+
+
 finish_test
@@ -0,0 +1,44 @@
+# 2017 April 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix indexexpr2
+
+do_execsql_test 1 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 'one');
+  INSERT INTO t1 VALUES(2, 'two');
+  INSERT INTO t1 VALUES(3, 'three');
+
+  CREATE INDEX i1 ON t1(b || 'x');
+}
+
+do_execsql_test 1.1 {
+  SELECT 'TWOX' == (b || 'x') FROM t1 WHERE (b || 'x')>'onex'
+} {0 0}
+
+do_execsql_test 1.2 {
+  SELECT 'TWOX' == (b || 'x') COLLATE nocase  FROM t1 WHERE (b || 'x')>'onex'
+} {0 1}
+
+do_execsql_test 2.0 {
+  CREATE INDEX i2 ON t1(a+1);
+}
+
+do_execsql_test 2.1 {
+  SELECT a+1, quote(a+1) FROM t1 ORDER BY 1;
+} {2 2 3 3 4 4}
+
+finish_test
+
@@ -356,6 +356,15 @@ do_execsql_test json-8.2 {
  SELECT a=json_extract(b,'$[0]') FROM t8;
 } {1}

+# 2017-04-12.  Regression reported on the mailing list by Rolf Ade
+#
+do_execsql_test json-8.3 {
+  SELECT json_valid(char(0x22,0xe4,0x22));
+} {1}
+do_execsql_test json-8.4 {
+  SELECT unicode(json_extract(char(0x22,228,0x22),'$'));
+} {228}
+
 # The json_quote() function transforms an SQL value into a JSON value.
 # String values are quoted and interior quotes are escaped.  NULL values
 # are rendered as the unquoted string "null".
@@ -688,5 +697,29 @@ do_execsql_test json-10.95 {
  SELECT json_valid('" \~ "');
 } {0}

+#--------------------------------------------------------------------------
+# 2017-04-11.  https://www.sqlite.org/src/info/981329adeef51011
+# Stack overflow on deeply nested JSON.
+#
+# The following tests confirm that deeply nested JSON is considered invalid.
+#
+do_execsql_test json-11.0 {
+  /* Shallow enough to be parsed */
+  SELECT json_valid(printf('%.2000c0%.2000c','[',']'));
+} {1}
+do_execsql_test json-11.1 {
+  /* Too deep by one */
+  SELECT json_valid(printf('%.2001c0%.2001c','[',']'));
+} {0}
+do_execsql_test json-11.2 {
+  /* Shallow enough to be parsed { */
+  SELECT json_valid(replace(printf('%.2000c0%.2000c','[','}'),'[','{"a":'));
+  /* } */
+} {1}
+do_execsql_test json-11.3 {
+  /* Too deep by one { */
+  SELECT json_valid(replace(printf('%.2001c0%.2001c','[','}'),'[','{"a":'));
+  /* } */
+} {0}

 finish_test
@@ -265,6 +265,8 @@ test_suite "fts3" -prefix "" -description {
  fts4incr.test fts4langid.test fts4lastrowid.test fts4merge2.test
  fts4merge4.test fts4merge.test fts4noti.test fts4onepass.test 
  fts4opt.test fts4unicode.test
+  fts3corrupt3.test
+  fts3misc.test
 }

 test_suite "fts5" -prefix "" -description {
@@ -0,0 +1,59 @@
+# 2017 April 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix pushdown
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  INSERT INTO t1 VALUES(1, 'b1', 'c1');
+  INSERT INTO t1 VALUES(2, 'b2', 'c2');
+  INSERT INTO t1 VALUES(3, 'b3', 'c3');
+  INSERT INTO t1 VALUES(4, 'b4', 'c4');
+  CREATE INDEX i1 ON t1(a, c);
+}
+
+proc f {val} {
+  lappend ::L $val
+  return 0
+}
+db func f f 
+
+do_test 1.1 {
+  set L [list]
+  execsql { SELECT * FROM t1 WHERE a=2 AND f(b) AND f(c) }
+  set L
+} {c2}
+
+do_test 1.2 {
+  set L [list]
+  execsql { SELECT * FROM t1 WHERE a=3 AND f(c) AND f(b) }
+  set L
+} {c3}
+
+do_execsql_test 1.3 {
+  DROP INDEX i1;
+  CREATE INDEX i1 ON t1(a, b);
+}
+do_test 1.4 {
+  set L [list]
+  execsql { SELECT * FROM t1 WHERE a=2 AND f(b) AND f(c) }
+  set L
+} {b2}
+
+do_test 1.5 {
+  set L [list]
+  execsql { SELECT * FROM t1 WHERE a=3 AND f(c) AND f(b) }
+  set L
+} {b3}
+  
+finish_test
@@ -39,6 +39,7 @@ set extras {
  REGISTER
  VECTOR
  SELECT_COLUMN
+  IF_NULL_ROW
  ASTERISK
  SPAN
  SPACE
@@ -250,7 +250,7 @@ static void StrAppend(Str *p, const char *z){
    sqlite3_uint64 nNew;
    if( p->oomErr ) return;
    nNew = p->nAlloc*2 + 100 + n;
-    zNew = sqlite3_realloc(p->z, nNew);
+    zNew = sqlite3_realloc(p->z, (int)nNew);
    if( zNew==0 ){
      sqlite3_free(p->z);
      memset(p, 0, sizeof(*p));
@@ -260,7 +260,7 @@ static void StrAppend(Str *p, const char *z){
    p->z = zNew;
    p->nAlloc = nNew;
  }
-  memcpy(p->z + p->n, z, n);
+  memcpy(p->z + p->n, z, (size_t)n);
  p->n += n;
  p->z[p->n] = 0;
 }
@@ -1116,15 +1116,15 @@ int main(int argc, char **argv){
        ** if the database file is the input being fuzzed, the SQL text is
        ** fuzzed at the same time. */
        if( sqlite3_table_column_metadata(db,0,"autoexec","sql",0,0,0,0,0)==0 ){
-          sqlite3_stmt *pStmt;
-          rc = sqlite3_prepare_v2(db, "SELECT sql FROM autoexec", -1, &pStmt, 0);
+          sqlite3_stmt *pStmt2;
+          rc = sqlite3_prepare_v2(db,"SELECT sql FROM autoexec",-1,&pStmt2,0);
          if( rc==SQLITE_OK ){
-            while( sqlite3_step(pStmt)==SQLITE_ROW ){
-              StrAppend(&sql, (const char*)sqlite3_column_text(pStmt, 0));
+            while( sqlite3_step(pStmt2)==SQLITE_ROW ){
+              StrAppend(&sql, (const char*)sqlite3_column_text(pStmt2, 0));
              StrAppend(&sql, "\n");
            }
          }
-          sqlite3_finalize(pStmt);
+          sqlite3_finalize(pStmt2);
          zSql = StrStr(&sql);
        }

@@ -168,12 +168,12 @@ static struct action *Action_new(void);
 static struct action *Action_sort(struct action *);

 /********** From the file "build.h" ************************************/
-void FindRulePrecedences();
-void FindFirstSets();
-void FindStates();
-void FindLinks();
-void FindFollowSets();
-void FindActions();
+void FindRulePrecedences(struct lemon*);
+void FindFirstSets(struct lemon*);
+void FindStates(struct lemon*);
+void FindLinks(struct lemon*);
+void FindFollowSets(struct lemon*);
+void FindActions(struct lemon*);

 /********* From the file "configlist.h" *********************************/
 void Configlist_init(void);
@@ -457,7 +457,7 @@ struct state *State_new(void);
 void State_init(void);
 int State_insert(struct state *, struct config *);
 struct state *State_find(struct config *);
-struct state **State_arrayof(/*  */);
+struct state **State_arrayof(void);

 /* Routines used for efficiency in Configlist_add */

@@ -561,8 +561,8 @@ void Action_add(
 ** default action for the state_number is returned.
 **
 ** All actions associated with a single state_number are first entered
-** into aLookahead[] using multiple calls to acttab_action().  Then the 
-** actions for that single state_number are placed into the aAction[] 
+** into aLookahead[] using multiple calls to acttab_action().  Then the
+** actions for that single state_number are placed into the aAction[]
 ** array with a single call to acttab_insert().  The acttab_insert() call
 ** also resets the aLookahead[] array in preparation for the next
 ** state number.
@@ -612,7 +612,7 @@ acttab *acttab_alloc(void){
  return p;
 }

-/* Add a new action to the current transaction set.  
+/* Add a new action to the current transaction set.
 **
 ** This routine is called once for each lookahead for a particular
 ** state.
@@ -674,7 +674,7 @@ int acttab_insert(acttab *p){
    }
  }

-  /* Scan the existing action table looking for an offset that is a 
+  /* Scan the existing action table looking for an offset that is a
  ** duplicate of the current transaction set.  Fall out of the loop
  ** if and when the duplicate is found.
  **
@@ -752,7 +752,7 @@ int acttab_insert(acttab *p){
 */

 /* Find a precedence symbol of every rule in the grammar.
-** 
+**
 ** Those rules which have a precedence symbol coded in the input
 ** grammar using the "[symbol]" construct will already have the
 ** rp->precsym field filled.  Other rules take as their precedence
@@ -1072,7 +1072,7 @@ void FindFollowSets(struct lemon *lemp)
      cfp->status = INCOMPLETE;
    }
  }
-  
+
  do{
    progress = 0;
    for(i=0; i<lemp->nstate; i++){
@@ -1103,7 +1103,7 @@ void FindActions(struct lemon *lemp)
  struct symbol *sp;
  struct rule *rp;

-  /* Add all of the reduce actions 
+  /* Add all of the reduce actions
  ** A reduce action is added for each element of the followset of
  ** a configuration which has its dot at the extreme right.
  */
@@ -1220,7 +1220,7 @@ static int resolve_conflict(
      apx->type = RD_RESOLVED;
    }
  }else{
-    assert( 
+    assert(
      apx->type==SH_RESOLVED ||
      apx->type==RD_RESOLVED ||
      apx->type==SSCONFLICT ||
@@ -1251,7 +1251,7 @@ static struct config *basis = 0;         /* Top of list of basis configs */
 static struct config **basisend = 0;     /* End of list of basis configs */

 /* Return a pointer to a new configuration */
-PRIVATE struct config *newconfig(){
+PRIVATE struct config *newconfig(void){
  struct config *newcfg;
  if( freelist==0 ){
    int i;
@@ -1277,7 +1277,7 @@ PRIVATE void deleteconfig(struct config *old)
 }

 /* Initialized the configuration list builder */
-void Configlist_init(){
+void Configlist_init(void){
  current = 0;
  currentend = &current;
  basis = 0;
@@ -1287,7 +1287,7 @@ void Configlist_init(){
 }

 /* Initialized the configuration list builder */
-void Configlist_reset(){
+void Configlist_reset(void){
  current = 0;
  currentend = &current;
  basis = 0;
@@ -1397,7 +1397,7 @@ void Configlist_closure(struct lemon *lemp)
 }

 /* Sort the configuration list */
-void Configlist_sort(){
+void Configlist_sort(void){
  current = (struct config*)msort((char*)current,(char**)&(current->next),
                                  Configcmp);
  currentend = 0;
@@ -1405,7 +1405,7 @@ void Configlist_sort(){
 }

 /* Sort the basis configuration list */
-void Configlist_sortbasis(){
+void Configlist_sortbasis(void){
  basis = (struct config*)msort((char*)current,(char**)&(current->bp),
                                Configcmp);
  basisend = 0;
@@ -1414,7 +1414,7 @@ void Configlist_sortbasis(){

 /* Return a pointer to the head of the configuration list and
 ** reset the list */
-struct config *Configlist_return(){
+struct config *Configlist_return(void){
  struct config *old;
  old = current;
  current = 0;
@@ -1424,7 +1424,7 @@ struct config *Configlist_return(){

 /* Return a pointer to the head of the configuration list and
 ** reset the list */
-struct config *Configlist_basis(){
+struct config *Configlist_basis(void){
  struct config *old;
  old = basis;
  basis = 0;
@@ -1466,7 +1466,7 @@ void ErrorMsg(const char *filename, int lineno, const char *format, ...){
 /* Report an out-of-memory condition and abort.  This function
 ** is used mostly by the "MemoryCheck" macro in struct.h
 */
-void memory_error(){
+void memory_error(void){
  fprintf(stderr,"Out of memory.  Aborting...\n");
  exit(1);
 }
@@ -1606,7 +1606,7 @@ int main(int argc, char **argv)
  OptInit(argv,options,stderr);
  if( version ){
     printf("Lemon version 1.0\n");
-     exit(0); 
+     exit(0);
  }
  if( OptNArgs()!=1 ){
    fprintf(stderr,"Exactly one filename argument is required.\n");
@@ -2051,7 +2051,7 @@ int OptInit(char **a, struct s_options *o, FILE *err)
  return 0;
 }

-int OptNArgs(){
+int OptNArgs(void){
  int cnt = 0;
  int dashdash = 0;
  int i;
@@ -2078,7 +2078,7 @@ void OptErr(int n)
  if( i>=0 ) errline(i,0,errstream);
 }

-void OptPrint(){
+void OptPrint(void){
  int i;
  int max, len;
  max = 0;
@@ -2307,7 +2307,7 @@ to follow the previous rule.");
    case IN_RHS:
      if( x[0]=='.' ){
        struct rule *rp;
-        rp = (struct rule *)calloc( sizeof(struct rule) + 
+        rp = (struct rule *)calloc( sizeof(struct rule) +
             sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1);
        if( rp==0 ){
          ErrorMsg(psp->filename,psp->tokenlineno,
@@ -2896,7 +2896,7 @@ void Parse(struct lemon *gp)
 static struct plink *plink_freelist = 0;

 /* Allocate a new plink */
-struct plink *Plink_new(){
+struct plink *Plink_new(void){
  struct plink *newlink;

  if( plink_freelist==0 ){
@@ -2997,7 +2997,7 @@ PRIVATE FILE *file_open(
  return fp;
 }

-/* Duplicate the input file without comments and without actions 
+/* Duplicate the input file without comments and without actions
 ** on rules */
 void Reprint(struct lemon *lemp)
 {
@@ -3148,7 +3148,7 @@ int PrintAction(
        indent,ap->sp->name,ap->x.rp->iRule);
      break;
    case SSCONFLICT:
-      fprintf(fp,"%*s shift        %-7d ** Parsing conflict **", 
+      fprintf(fp,"%*s shift        %-7d ** Parsing conflict **",
        indent,ap->sp->name,ap->x.stp->statenum);
      break;
    case SH_RESOLVED:
@@ -3421,7 +3421,7 @@ PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno)
    (*lineno)++;
  }
  if (!lemp->nolinenosflag) {
-    (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); 
+    (*lineno)++; tplt_linedir(out,*lineno,lemp->outname);
  }
  return;
 }
@@ -3466,8 +3466,8 @@ void emit_destructor_code(
   fputc(*cp,out);
 }
 fprintf(out,"\n"); (*lineno)++;
- if (!lemp->nolinenosflag) { 
-   (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); 
+ if (!lemp->nolinenosflag) {
+   (*lineno)++; tplt_linedir(out,*lineno,lemp->outname);
 }
 fprintf(out,"}\n"); (*lineno)++;
 return;
@@ -3590,7 +3590,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
    /* There is no LHS value symbol. */
    lhsdirect = 1;
  }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){
-    /* The LHS symbol and the left-most RHS symbol are the same, so 
+    /* The LHS symbol and the left-most RHS symbol are the same, so
    ** direct writing is allowed */
    lhsdirect = 1;
    lhsused = 1;
@@ -3601,7 +3601,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
        "different datatypes.",
        rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]);
      lemp->errorcnt++;
-    }    
+    }
  }else{
    lemon_sprintf(zOvwrt, "/*%s-overwrites-%s*/",
                  rp->lhsalias, rp->rhsalias[0]);
@@ -3740,7 +3740,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
  return rc;
 }

-/* 
+/*
 ** Generate code which executes when the rule "rp" is reduced.  Write
 ** the code to "out".  Make sure lineno stays up-to-date.
 */
@@ -4161,7 +4161,7 @@ void ReportTable(
  for(i=0; i<lemp->nxstate; i++){
    for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){
      if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){
-        ap->x.rp->doesReduce = i;
+        ap->x.rp->doesReduce = 1;
      }
    }
  }
@@ -4237,7 +4237,7 @@ void ReportTable(
  fprintf(out, "#define YY_SHIFT_COUNT    (%d)\n", n-1); lineno++;
  fprintf(out, "#define YY_SHIFT_MIN      (%d)\n", mnTknOfst); lineno++;
  fprintf(out, "#define YY_SHIFT_MAX      (%d)\n", mxTknOfst); lineno++;
-  fprintf(out, "static const %s yy_shift_ofst[] = {\n", 
+  fprintf(out, "static const %s yy_shift_ofst[] = {\n",
       minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz));
       lineno++;
  lemp->tablesize += n*sz;
@@ -4264,7 +4264,7 @@ void ReportTable(
  fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++;
  fprintf(out, "#define YY_REDUCE_MIN   (%d)\n", mnNtOfst); lineno++;
  fprintf(out, "#define YY_REDUCE_MAX   (%d)\n", mxNtOfst); lineno++;
-  fprintf(out, "static const %s yy_reduce_ofst[] = {\n", 
+  fprintf(out, "static const %s yy_reduce_ofst[] = {\n",
          minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++;
  lemp->tablesize += n*sz;
  for(i=j=0; i<n; i++){
@@ -4343,7 +4343,7 @@ void ReportTable(
  tplt_xfer(lemp->name,in,out,&lineno);

  /* Generate code which executes every time a symbol is popped from
-  ** the stack while processing errors or while destroying the parser. 
+  ** the stack while processing errors or while destroying the parser.
  ** (In other words, generate the %destructor actions)
  */
  if( lemp->tokendest ){
@@ -4409,7 +4409,7 @@ void ReportTable(
  tplt_print(out,lemp,lemp->overflow,&lineno);
  tplt_xfer(lemp->name,in,out,&lineno);

-  /* Generate the table of rule information 
+  /* Generate the table of rule information
  **
  ** Note: This code depends on the fact that rules are number
  ** sequentually beginning with 0.
@@ -4520,7 +4520,7 @@ void ReportHeader(struct lemon *lemp)
    for(i=1; i<lemp->nterminal; i++){
      fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i);
    }
-    fclose(out);  
+    fclose(out);
  }
  return;
 }
@@ -4567,7 +4567,7 @@ void CompressTables(struct lemon *lemp)
        rbest = rp;
      }
    }
- 
+
    /* Do not make a default if the number of rules to default
    ** is not at least 1 or if the wildcard token is a possible
    ** lookahead.
@@ -4724,7 +4724,7 @@ void SetSize(int n)
 }

 /* Allocate a new set */
-char *SetNew(){
+char *SetNew(void){
  char *s;
  s = (char*)calloc( size, 1);
  if( s==0 ){
@@ -4830,7 +4830,7 @@ typedef struct s_x1node {
 static struct s_x1 *x1a;

 /* Allocate a new associative array */
-void Strsafe_init(){
+void Strsafe_init(void){
  if( x1a ) return;
  x1a = (struct s_x1*)malloc( sizeof(struct s_x1) );
  if( x1a ){
@@ -4997,7 +4997,7 @@ typedef struct s_x2node {
 static struct s_x2 *x2a;

 /* Allocate a new associative array */
-void Symbol_init(){
+void Symbol_init(void){
  if( x2a ) return;
  x2a = (struct s_x2*)malloc( sizeof(struct s_x2) );
  if( x2a ){
@@ -5194,7 +5194,7 @@ typedef struct s_x3node {
 static struct s_x3 *x3a;

 /* Allocate a new associative array */
-void State_init(){
+void State_init(void){
  if( x3a ) return;
  x3a = (struct s_x3*)malloc( sizeof(struct s_x3) );
  if( x3a ){
@@ -5288,7 +5288,7 @@ struct state *State_find(struct config *key)
 /* Return an array of pointers to all data in the table.
 ** The array is obtained from malloc.  Return NULL if memory allocation
 ** problems, or if the array is empty. */
-struct state **State_arrayof()
+struct state **State_arrayof(void)
 {
  struct state **array;
  int i,arrSize;
@@ -5334,7 +5334,7 @@ typedef struct s_x4node {
 static struct s_x4 *x4a;

 /* Allocate a new associative array */
-void Configtable_init(){
+void Configtable_init(void){
  if( x4a ) return;
  x4a = (struct s_x4*)malloc( sizeof(struct s_x4) );
  if( x4a ){
@@ -12,6 +12,7 @@

 #if !defined(_MSC_VER)
 #include <unistd.h>
+#include <sys/types.h>
 #else
 #include <io.h>
 #endif
@@ -579,6 +580,14 @@ int main(int argc, char **argv){
        decode_btree_page(a, iStart, hdrSize, zLeft+1);
        free(a);
        continue;
+#if !defined(_MSC_VER)
+      }else if( zLeft && strcmp(zLeft,"truncate")==0 ){
+        /* Frame number followed by "truncate" truncates the WAL file
+        ** after that frame */
+        off_t newSize = 32 + iStart*(pagesize+24);
+        truncate(argv[1], newSize);
+        continue;
+#endif
      }else{
        iEnd = iStart;
      }
Author	SHA1	Message	Date
drh	6ab3eb5d0e	Minor size and performance improvements to the push-down optimization. FossilOrigin-Name: 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d	2017-04-29 14:56:55 +00:00
dan	6f654a40e8	Within a loop that uses a non-covering index test, test non-indexed terms that can be tested without seeking the main table cursor before those that cannot. FossilOrigin-Name: afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444	2017-04-28 19:59:55 +00:00
dan	4b17369579	Add new test file cachespill.test. FossilOrigin-Name: 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2	2017-04-26 17:21:33 +00:00
dan	b9e6a73866	Replace <fts5.h> with "fts5.h" in test file fts5_test_tok.c. FossilOrigin-Name: 63d9ca5c7392e1efe3330689fe750310a952287e843b3242178724e8561fae0e	2017-04-26 14:34:04 +00:00
drh	69bfa55279	Fix a subtle bug in Lemon discovered and reported on the mailing list by Kelvin Sherlock, who also suggested the correct fix. FossilOrigin-Name: 304689f8acb53d68e1afed6e6c4332e78132e3d57071b8f94df0d13515b3b3d8	2017-04-26 04:32:17 +00:00
dan	a1188d6d93	When building an ephemeral table of integers, do not assume that the table does not already contain N if the insert cursor points to N-1. Fix for ticket [61fe9745]. FossilOrigin-Name: 1c1b0862236f564283be08338790d6494e705f1198547a59499e3f8c07f2cd99	2017-04-24 14:16:55 +00:00
drh	0c3233be0b	Fix an assertion fault found by OSSFuzz. FossilOrigin-Name: e39769f442ea86273f8fb7bd2f249b7d760b2fe4bf9ec191ff91def8231f3f35	2017-04-22 00:20:49 +00:00
dan	39c9d3a367	Remove a NEVER macro for a condition that may be true as of [a47efb7c]. Problem reported by OSSFuzz. FossilOrigin-Name: bdc50d8d127266d02b291f6bfcd4e35eb07994ce23987d6e3921124cd881929a	2017-04-21 17:03:32 +00:00
dan	0619838c55	Fix an FTS5 bug that could cause a prefix-query without a prefix-index on a database that contains delete-markers to return extra, non-matching, rows. FossilOrigin-Name: 840042cb2bed2924e2263f21887317f661e5a585c4466c3af25b91ed57a6b49b	2017-04-21 16:04:18 +00:00
dan	dc62daca86	Further improvements to coverage of fts3 module. FossilOrigin-Name: 6b21d0fdebdccfaf63590d9ca9a279c22b8baec07c1a669b9f617f25bd857384	2017-04-19 13:25:45 +00:00
dan	a059e99ca8	Further modifications and test cases to improve test coverage of fts3. FossilOrigin-Name: ea8a0d2ce0cb1ca3f4f18c72fb780d1c26792799acc87e6726f9eaccf2f178bf	2017-04-19 07:33:52 +00:00
dan	0027e98b85	Use sqlite3_table_column_metadata() instead of a SELECT statement to check for the existence of a %_stat table in fts3. This leads to smaller and easier to test code. FossilOrigin-Name: dc2a48020a3c856a8b4cfa2309c290f0508b991e92cc0e4de680d19aae7872fc	2017-04-18 13:50:00 +00:00
drh	bff6fe1f40	Improved optimizations of views as the right operand of a LEFT JOIN. FossilOrigin-Name: 41c27bc0ff1d3135cdb6273ede4595f5bb0c0e1e1d470ea1633cb525674cf431	2017-04-18 11:20:19 +00:00
dan	bda00fbdb7	Refactor the fts3ColumnMethod() function so that all branches can be covered. FossilOrigin-Name: e47fdb493bd76d85f6f05771ee7a0b3ee31b1eb05839a60d2bdb47149ac692d8	2017-04-18 05:49:23 +00:00
drh	cfcca02765	Defer checking for null in the comparison operators, since that is an uncommon case. FossilOrigin-Name: 5684525613961fed9db6a4d10dbe25521201b24f08e011df3a20ac895316957d	2017-04-17 23:23:17 +00:00
drh	e2bc6552fe	Do not allow a Mem object to be both NULL and some other type at the same time. FossilOrigin-Name: e698db1956bb3aba32cd3ec633ec20f5d19b1a10bc68d3772903bca3c87ee158	2017-04-17 20:50:34 +00:00
dan	4bee5599c0	Consolidate two branches of code in the "PRAGMA foreign_key_check" implementation. FossilOrigin-Name: 69f51f838abcf57b35e41f7a10fbb45f56536f93825aac865debc3c8315930be	2017-04-17 18:42:33 +00:00
dan	940464b1dc	Fix a problem in "PRAGMA foreign_key_check" in handling a WITHOUT ROWID child table with an INTEGER PRIMARY KEY parent key. Also, if an FK violation is detected in a WITHOUT ROWID child table, do not try to read and return the rowid. The third column returned by "PRAGMA foreign_key_check" in this case (WITHOUT ROWID child table) is now always set to NULL. FossilOrigin-Name: 690870bd7b2e607b7992c4496c9f08c29eb72a36af002c606fd7aa3dcf94a2a4	2017-04-17 18:02:41 +00:00
dan	624edacf6a	Further improvements to test coverage in fts3. FossilOrigin-Name: 352413eed469802665e7d2c17b7fe6e3a0b0b2209ce45bdc85fd4243cec50ea6	2017-04-17 16:07:25 +00:00
dan	ddf4747053	Improve coverage of code in fts3. FossilOrigin-Name: 1a08a836fa919524216a16faf5212eebc9ad788c9e15463b68b0576dfccd4387	2017-04-17 13:38:09 +00:00
drh	6887e8facd	Fix the ".column" output mode in the command-line shell so that it correctly counts and formats multi-byte UTF characters. FossilOrigin-Name: f508aff8d1782abdff4d03726ae098607a0ee6cfd2e35b130097ee5043e98960	2017-04-17 13:18:42 +00:00
drh	b6d5a43b8a	In the showwal command-line tool, for unix builds, if the auxiliary argument is of the form "Ntruncate" where "N" is a frame number, then truncate the WAL file after the N-th frame. FossilOrigin-Name: 90015df30655d884ecf7ae61e588824696954252dc6b1a5f78cf2de8cb236104	2017-04-16 22:41:49 +00:00
drh	b65b3e429a	Add the anycollseq.c loadable extension in etc/misc FossilOrigin-Name: d7b9813cb17615c3d00afd6994a4309d6d48c8e924b6cd813c543e1fa65c7719	2017-04-16 22:08:31 +00:00
drh	147fd88268	In the skip-ahead-distinct optimization, fix a bug in the logic that determines when to invoke the optimization based on sqlite_stat1 statistics. FossilOrigin-Name: 89f9e4363aa19f306e55f749c442eae2f8994f6a47c65e645a79b308b450d5e5	2017-04-15 11:53:47 +00:00
drh	a2e2d92b21	Fix to the decision logic for when to use the skip-ahead-distinct optimization. FossilOrigin-Name: e50fd48969f99bc988389c53ff46714603b1d11de12c71b55c00cbee037f073c	2017-04-14 22:41:27 +00:00
drh	06f60d8e87	Remove end-of-line whitespace from lemon.c. FossilOrigin-Name: d78355c85f49e139f1ac1a660563865350f0e442652b7bc50b684398f81cc602	2017-04-14 19:46:12 +00:00
drh	14d8855b8f	Fix some left-over K&R-C constructs in lemon.c. No changes to the core. FossilOrigin-Name: a53799059d4ece246610b6c877ab7088ded3548cbca7149a03deea8ac0e27e68	2017-04-14 19:44:15 +00:00
drh	31d6fd5507	Add the TK_IF_NULL_ROW opcode to deal with non-NULL result columns in the result set of a view or subquery on the RHS of a LEFT JOIN that gets flattened. FossilOrigin-Name: 3a5860d86fadcf924316707918bf283d26c53b1473e5e67f5cff59d18c2a7742	2017-04-14 19:03:10 +00:00
drh	1a17b44231	When doing a DISTINCT query using an index, try to use the index to skip ahead to the next distinct element, rather than doing a full scan of the index. (This is the "skip-ahead-distinct" optimization.) FossilOrigin-Name: f489b5bb6b35665befdd411c2c55df5258e83cba265d8c4427ba22529cf882a4	2017-04-14 17:30:43 +00:00
drh	399c7e21e4	An initial attempt to optimize VIEWs that occur as the right operand of a LEFT JOIN. This particular check-in does not work correctly because it does not deal with the case of columns in the VIEW that return non-NULL even when all columns in the table of the VIEW are NULL because of the LEFT JOIN. FossilOrigin-Name: 1838a59c8a1c151bd6fc822b0ffef661803cf0e4704c917e74a04567b81740b9	2017-04-14 17:18:45 +00:00
mistachkin	b1ed717f18	Make USE_FULLWARN=1 the default for MSVC and fix harmless compiler warnings. FossilOrigin-Name: 6bf673767b8e5cedef1acff795cbe524fab8db2525c06424db4e038934a33936	2017-04-14 14:50:34 +00:00
drh	d97cda43ab	Enhance the sqlite3TreeView() display for Expr objects so that it shows the iRightJoinTable value for Expr nodes that have the EP_FromJoin property. FossilOrigin-Name: 5159cb8f2bcfb1f7114786ba23082d2b91a26e3a7ddfae75f8bd362792949d5e	2017-04-14 14:02:14 +00:00
drh	46967de2ec	Simplify the interface to the subst() routines that are part of the query flattener by collecting common parameters into the SubstContext object and passing around a pointer to that object. FossilOrigin-Name: e651074be28b2077b8a298f513e2e699ceca99164ace9869ea50873c0637fd2e	2017-04-14 12:39:37 +00:00
drh	7c88bcd473	Remove an incorrect ALWAYS(). FossilOrigin-Name: f956f6ae6b0863c70c4a2b227338d5c332e87f2ec6f3fcdf2233c14c5b78ca2b	2017-04-14 12:27:45 +00:00
drh	571311502d	Fix a couple of unreachable branches. FossilOrigin-Name: 1aa0ea8db7580aff8bed2f78117ed50019e5545cef48cab782f512a2599fd0c5	2017-04-14 00:45:51 +00:00
drh	172806e489	Fix the skip-ahead-distinct optimization so that it works with indexes that have repeated columns with different collating sequences. FossilOrigin-Name: ce1e2b88777e00a82c04abe5ba35eec81b5f324e462f099cd00b21054f369688	2017-04-13 21:29:02 +00:00
drh	c04ea80f64	Simplification of the skip-ahead-distinct logic. There is still an issue with handling COLLATE. FossilOrigin-Name: 57c5173b6376a76013b0611ce9770100bd8c04e80402e35d821d8614709f4795	2017-04-13 19:48:29 +00:00
dan	a74f5c2923	New test cases and minor fixes for the optimization on this branch. FossilOrigin-Name: 70ac9ea1a6cb2f4906f00f1a04f668e5ba5eeed8d4d0fa4be57a9c9eb0683697	2017-04-13 18:33:33 +00:00
dan	d0a537100c	Reinstate the SQLITE_API qualifier on the sqlite3_delete_database() method in test_delete.c. Accidentally removed by the previous commit. FossilOrigin-Name: 59c70108fd179932ccdd860f93e1cd68b77476d3b1af1af00cf6e378c9984862	2017-04-13 15:51:27 +00:00
dan	44ae27ac32	Update the code in test_delete.c to use the "win32" VFS if SQLITE_OS_WIN is defined. FossilOrigin-Name: fa9bb7b768027677f7e7d5a196ba5b245dfc8d8986ccd101c8dab671bd15719d	2017-04-13 15:36:47 +00:00
drh	839fa6d814	Only use the skip-ahead-distinct optimization if the index has been analyzed and we know that a skip-head is likely to skip over at least 11 rows. The magic number 11 was determined by experimentation. FossilOrigin-Name: 0cf16decd534bf2d66620c293f8c8987f356305f2d97f8fd12d260bda1571385	2017-04-13 13:01:59 +00:00
dan	929b695111	Allow a user column name to be used on the LHS of a MATCH operator in FTS5. FossilOrigin-Name: 6f54ffd151b0eca6f9ef57ac54802584a839cfc7373f10c100fc18c855edcc0a	2017-04-13 09:45:21 +00:00
drh	8489bf5aff	Forward port the skip-ahead-distinct branch which was abandoned for some reason that I do not recall. This port should have been achived by a merge of trunk into the previous head of skip-ahead-distinct, but that did not work. So I had to manually "rebase" the changes. FossilOrigin-Name: 132339a1fb0b9664df4d3eefbed6e77ef100ba95a91dcc622da7bd3fcdfcd6af	2017-04-13 01:19:30 +00:00
drh	8671538cad	Fix a regression caused by the fix for ticket [6c9b5514077fed34551f98e64c09a1] - control characters allowed in JSON. FossilOrigin-Name: 8e7b611863247a3bf46a96ec4b47d24c0ae0d60c9cee968a1cfd1da157e7c9bb	2017-04-13 00:12:32 +00:00
dan	03155f659f	Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well as "<colset> : NEAR(...)" and "<colset> : <phrase>". FossilOrigin-Name: c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396	2017-04-12 17:50:12 +00:00
drh	26849e14c2	Improved \n and \r escapes in the ext/misc/dbdump.c utility function. The implementation of dbdump.c now matches the implementation in the CLI. FossilOrigin-Name: f2643315bb41a71eebd79f5d671f9163187e299a52ff8a481186f1e8fa7e5262	2017-04-12 17:38:24 +00:00
dan	15e0f3d901	Avoid updating unaffected indexes on a table as part of an UPDATE that requires foreign key processing in some cases. FossilOrigin-Name: 7aae5c0f99aa2fda85654242cfc9e23a0f981d9ce4ab17610d619cd208540b3d	2017-04-11 20:48:30 +00:00
dan	940b5eaa5e	Avoid updating unaffected indexes on a table as part of an UPDATE that requires foreign key processing in some cases. FossilOrigin-Name: 477bea9ed0dd0fa69896bfd16d9b1d22170cbab784e3279ce65c29c47e032f34	2017-04-11 19:58:55 +00:00
drh	ff6d50e973	Limit the depth of recursion for valid JSON in the JSON1 extension in order to avoid using excess stack space in the recursive descent parser. Fix for ticket [981329adeef51011052667a9]. FossilOrigin-Name: 1f68c184596912d742b50b1ca38252a9e783aacf121619a27b17a7ae9f6df041	2017-04-11 18:55:05 +00:00
drh	e97c9ff49a	Smaller and faster implementation of exprMightBeIndexed(). FossilOrigin-Name: 76cd611d41465fcec61c21520d55172cb236530f38386b7d4a5544ba87de2353	2017-04-11 18:06:48 +00:00
drh	ef90a6b8dd	Very slight smaller and faster sqlite3SelectNew() FossilOrigin-Name: 4143650c4ce32289d2301cdfc69bb10877246420f656bed122886f6803fc956a	2017-04-11 16:44:39 +00:00
drh	cbae543a5c	Add an ALWAYS() around an unreachable condition in sqlite3VdbeMemGrow(). FossilOrigin-Name: 0f3eb61f44c94234ffc14c3a493a27ec62c0d1e2bb62798f36eafa98ddcdc58e	2017-04-11 12:20:54 +00:00
dan	b8d29c2f75	Fix a segfault that could occur if an indexed expression was used in a comparison operation within the result-set of a SELECT statement. FossilOrigin-Name: d6bb7c42ff6309ce168ccdcf03b4cdabfccfc9e2a911d254ac7dc4fea4aa2bc1	2017-04-11 11:52:25 +00:00
drh	d6e992cb1e	When using an index on an expression, try to use the columns of the index rather than pulling columns from the table and reevaluating the expression. FossilOrigin-Name: a47efb7c8520a01110ce3b3531ebe1bab6720780d67fba001992c44c5807d332	2017-04-11 01:30:42 +00:00