From 795502d26c770c06b1d8d586fd316a6e374ed1d6 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 22 Jan 2016 19:48:34 +0000
Subject: [PATCH 1/6] Experimental performance enhancements for fts5.

FossilOrigin-Name: b5a57b812fd6a734cf2a342bf0b730ae18912d73
---
 ext/fts5/fts5Int.h        |  10 +-
 ext/fts5/fts5_expr.c      |  42 ++--
 ext/fts5/fts5_index.c     | 389 +++++++++++++++++++++++++++-----------
 ext/fts5/fts5_vocab.c     |  56 +++---
 ext/fts5/test/fts5ac.test |   1 -
 manifest                  |  25 +--
 manifest.uuid             |   2 +-
 7 files changed, 355 insertions(+), 170 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index b2f0d6c34e..7c50da9656 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -315,6 +315,12 @@ void sqlite3Fts5TermsetFree(Fts5Termset*);
 typedef struct Fts5Index Fts5Index;
 typedef struct Fts5IndexIter Fts5IndexIter;
 
+struct Fts5IndexIter {
+  i64 iRowid;
+  const u8 *pData;
+  int nData;
+};
+
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
@@ -396,6 +402,8 @@ void sqlite3Fts5IterClose(Fts5IndexIter*);
 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
 
+int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);
+
 
 /*
 ** Insert or remove data to or from the index. Each time a document is 
@@ -469,8 +477,6 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
-int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);
-
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index c51ed7939c..290baf4e2c 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -323,11 +323,15 @@ static int fts5ExprSynonymList(
       const u8 *a;
       int n;
 
-      if( bCollist ){
+      if( 0 && bCollist ){
         rc = sqlite3Fts5IterCollist(pIter, &a, &n);
       }else{
         i64 dummy;
+#if 0
         rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
+#endif
+        a = pIter->pData;
+        n = pIter->nData;
       }
 
       if( rc!=SQLITE_OK ) goto synonym_poslist_out;
@@ -436,7 +440,12 @@ static int fts5ExprPhraseIsMatch(
           pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
       );
     }else{
+      Fts5IndexIter *pIter = pTerm->pIter;
+#if 0
       rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
+#endif
+      a = pIter->pData;
+      n = pIter->nData;
     }
     if( rc!=SQLITE_OK ) goto ismatch_out;
     sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
@@ -775,6 +784,7 @@ static int fts5ExprNearTest(
     for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
       Fts5IndexIter *pIter = pTerm->pIter;
       if( sqlite3Fts5IterEof(pIter)==0 ){
+#if 0
         int n;
         i64 iRowid;
         rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
@@ -784,6 +794,10 @@ static int fts5ExprNearTest(
         }else if( iRowid==pNode->iRowid && n>0 ){
           pPhrase->poslist.n = 1;
         }
+#endif
+        if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){
+          pPhrase->poslist.n = 1;
+        }
       }
     }
     return pPhrase->poslist.n;
@@ -800,9 +814,13 @@ static int fts5ExprNearTest(
         rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
         if( bMatch==0 ) break;
       }else{
+        Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
+        fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData);
+#if 0
         rc = sqlite3Fts5IterPoslistBuffer(
             pPhrase->aTerm[0].pIter, &pPhrase->poslist
         );
+#endif
       }
     }
 
@@ -823,21 +841,20 @@ static int fts5ExprTokenTest(
   ** fts5_index.c iterator object. This is much faster than synthesizing 
   ** a new poslist the way we have to for more complicated phrase or NEAR
   ** expressions.  */
-  Fts5ExprNearset *pNear = pNode->pNear;
-  Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
+  Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
   Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
-  Fts5Colset *pColset = pNear->pColset;
-  int rc;
 
   assert( pNode->eType==FTS5_TERM );
-  assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
+  assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 );
   assert( pPhrase->aTerm[0].pSynonym==0 );
 
-  rc = sqlite3Fts5IterPoslist(pIter, pColset, 
-      (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid
-  );
+  pPhrase->poslist.n = pIter->nData;
+  if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){
+    pPhrase->poslist.p = (u8*)pIter->pData;
+  }
+  pNode->iRowid = pIter->iRowid;
   pNode->bNomatch = (pPhrase->poslist.n==0);
-  return rc;
+  return SQLITE_OK;
 }
 
 /*
@@ -2484,6 +2501,8 @@ int sqlite3Fts5ExprPhraseCollist(
   int rc = SQLITE_OK;
 
   assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
+  assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
+
   if( pNode->bEof==0 
    && pNode->iRowid==pExpr->pRoot->iRowid 
    && pPhrase->poslist.n>0
@@ -2503,7 +2522,8 @@ int sqlite3Fts5ExprPhraseCollist(
         *ppCollist = a;
       }
     }else{
-      sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
+      *ppCollist = pPhrase->aTerm[0].pIter->pData;
+      *pnCollist = pPhrase->aTerm[0].pIter->nData;
     }
   }else{
     *ppCollist = 0;
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index fc11a23413..2ea57b04f9 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -261,6 +261,7 @@ typedef struct Fts5Data Fts5Data;
 typedef struct Fts5DlidxIter Fts5DlidxIter;
 typedef struct Fts5DlidxLvl Fts5DlidxLvl;
 typedef struct Fts5DlidxWriter Fts5DlidxWriter;
+typedef struct Fts5Iter Fts5Iter;
 typedef struct Fts5PageWriter Fts5PageWriter;
 typedef struct Fts5SegIter Fts5SegIter;
 typedef struct Fts5DoclistIter Fts5DoclistIter;
@@ -503,10 +504,16 @@ struct Fts5SegIter {
 **   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
 **   There is no way to tell if this is populated or not.
 */
-struct Fts5IndexIter {
+struct Fts5Iter {
+  Fts5IndexIter base;             /* Base class containing output vars */
+
   Fts5Index *pIndex;              /* Index that owns this iterator */
   Fts5Structure *pStruct;         /* Database structure for this iterator */
   Fts5Buffer poslist;             /* Buffer containing current poslist */
+  Fts5Colset *pColset;            /* Restrict matches to these columns */
+
+  /* Invoked to set output variables. */
+  void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
 
   int nSeg;                       /* Size of aSeg[] array */
   int bRev;                       /* True to iterate in reverse order */
@@ -1752,7 +1759,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
 ** points to a delete marker. A delete marker is an entry with a 0 byte
 ** position-list.
 */
-static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){
+static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){
   Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
   return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
 }
@@ -2406,7 +2413,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){
 ** two iterators.
 */
 static void fts5AssertComparisonResult(
-  Fts5IndexIter *pIter, 
+  Fts5Iter *pIter, 
   Fts5SegIter *p1,
   Fts5SegIter *p2,
   Fts5CResult *pRes
@@ -2447,7 +2454,7 @@ static void fts5AssertComparisonResult(
 ** statement used to verify that the contents of the pIter->aFirst[] array
 ** are correct.
 */
-static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
+static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){
   if( p->rc==SQLITE_OK ){
     Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
     int i;
@@ -2492,7 +2499,7 @@ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
 ** to a key that is a duplicate of another, higher priority, 
 ** segment-iterator in the pSeg->aSeg[] array.
 */
-static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){
+static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
   int i1;                         /* Index of left-hand Fts5SegIter */
   int i2;                         /* Index of right-hand Fts5SegIter */
   int iRes;
@@ -2638,7 +2645,7 @@ static void fts5SegIterNextFrom(
 /*
 ** Free the iterator object passed as the second argument.
 */
-static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){
+static void fts5MultiIterFree(Fts5Index *p, Fts5Iter *pIter){
   if( pIter ){
     int i;
     for(i=0; i<pIter->nSeg; i++){
@@ -2652,7 +2659,7 @@ static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){
 
 static void fts5MultiIterAdvanced(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
-  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
+  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
   int iChanged,                   /* Index of sub-iterator just advanced */
   int iMinset                     /* Minimum entry in aFirst[] to set */
 ){
@@ -2680,8 +2687,9 @@ static void fts5MultiIterAdvanced(
 */ 
 static int fts5MultiIterAdvanceRowid(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
-  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
-  int iChanged                    /* Index of sub-iterator just advanced */
+  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
+  int iChanged,                   /* Index of sub-iterator just advanced */
+  Fts5SegIter **ppFirst
 ){
   Fts5SegIter *pNew = &pIter->aSeg[iChanged];
 
@@ -2714,13 +2722,14 @@ static int fts5MultiIterAdvanceRowid(
     }
   }
 
+  *ppFirst = pNew;
   return 0;
 }
 
 /*
 ** Set the pIter->bEof variable based on the state of the sub-iterators.
 */
-static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
+static void fts5MultiIterSetEof(Fts5Iter *pIter){
   Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
   pIter->bEof = pSeg->pLeaf==0;
   pIter->iSwitchRowid = pSeg->iRowid;
@@ -2735,39 +2744,44 @@ static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
 */
 static void fts5MultiIterNext(
   Fts5Index *p, 
-  Fts5IndexIter *pIter,
+  Fts5Iter *pIter,
   int bFrom,                      /* True if argument iFrom is valid */
   i64 iFrom                       /* Advance at least as far as this */
 ){
-  if( p->rc==SQLITE_OK ){
-    int bUseFrom = bFrom;
-    do {
-      int iFirst = pIter->aFirst[1].iFirst;
-      int bNewTerm = 0;
-      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
-      assert( p->rc==SQLITE_OK );
-      if( bUseFrom && pSeg->pDlidx ){
-        fts5SegIterNextFrom(p, pSeg, iFrom);
-      }else{
-        pSeg->xNext(p, pSeg, &bNewTerm);
-      }
+  int bUseFrom = bFrom;
+  while( p->rc==SQLITE_OK ){
+    int iFirst = pIter->aFirst[1].iFirst;
+    int bNewTerm = 0;
+    Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
+    assert( p->rc==SQLITE_OK );
+    if( bUseFrom && pSeg->pDlidx ){
+      fts5SegIterNextFrom(p, pSeg, iFrom);
+    }else{
+      pSeg->xNext(p, pSeg, &bNewTerm);
+    }
 
-      if( pSeg->pLeaf==0 || bNewTerm 
-       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
-      ){
-        fts5MultiIterAdvanced(p, pIter, iFirst, 1);
-        fts5MultiIterSetEof(pIter);
-      }
-      fts5AssertMultiIterSetup(p, pIter);
+    if( pSeg->pLeaf==0 || bNewTerm 
+     || fts5MultiIterAdvanceRowid(p, pIter, iFirst, &pSeg)
+    ){
+      fts5MultiIterAdvanced(p, pIter, iFirst, 1);
+      fts5MultiIterSetEof(pIter);
+      pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
+      if( pSeg->pLeaf==0 ) return;
+    }
 
-      bUseFrom = 0;
-    }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) );
+    fts5AssertMultiIterSetup(p, pIter);
+    assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf );
+    if( pIter->bSkipEmpty==0 || pSeg->nPos ){
+      pIter->xSetOutputs(pIter, pSeg);
+      return;
+    }
+    bUseFrom = 0;
   }
 }
 
 static void fts5MultiIterNext2(
   Fts5Index *p, 
-  Fts5IndexIter *pIter,
+  Fts5Iter *pIter,
   int *pbNewTerm                  /* OUT: True if *might* be new term */
 ){
   assert( pIter->bSkipEmpty );
@@ -2780,7 +2794,7 @@ static void fts5MultiIterNext2(
       assert( p->rc==SQLITE_OK );
       pSeg->xNext(p, pSeg, &bNewTerm);
       if( pSeg->pLeaf==0 || bNewTerm 
-       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
+       || fts5MultiIterAdvanceRowid(p, pIter, iFirst, &pSeg)
       ){
         fts5MultiIterAdvanced(p, pIter, iFirst, 1);
         fts5MultiIterSetEof(pIter);
@@ -2794,17 +2808,19 @@ static void fts5MultiIterNext2(
   }
 }
 
+static void fts5IterSetOutputs_Noop(Fts5Iter *pIter, Fts5SegIter *pSeg){
+}
 
-static Fts5IndexIter *fts5MultiIterAlloc(
+static Fts5Iter *fts5MultiIterAlloc(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   int nSeg
 ){
-  Fts5IndexIter *pNew;
+  Fts5Iter *pNew;
   int nSlot;                      /* Power of two >= nSeg */
 
   for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
   pNew = fts5IdxMalloc(p, 
-      sizeof(Fts5IndexIter) +             /* pNew */
+      sizeof(Fts5Iter) +                  /* pNew */
       sizeof(Fts5SegIter) * (nSlot-1) +   /* pNew->aSeg[] */
       sizeof(Fts5CResult) * nSlot         /* pNew->aFirst[] */
   );
@@ -2812,12 +2828,13 @@ static Fts5IndexIter *fts5MultiIterAlloc(
     pNew->nSeg = nSlot;
     pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
     pNew->pIndex = p;
+    pNew->xSetOutputs = fts5IterSetOutputs_Noop;
   }
   return pNew;
 }
 
 /*
-** Allocate a new Fts5IndexIter object.
+** Allocate a new Fts5Iter object.
 **
 ** The new object will be used to iterate through data in structure pStruct.
 ** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
@@ -2835,14 +2852,14 @@ static void fts5MultiIterNew(
   const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
   int iLevel,                     /* Level to iterate (-1 for all) */
   int nSegment,                   /* Number of segments to merge (iLevel>=0) */
-  Fts5IndexIter **ppOut           /* New object */
+  Fts5Iter **ppOut                /* New object */
 ){
   int nSeg = 0;                   /* Number of segment-iters in use */
   int iIter = 0;                  /* */
   int iSeg;                       /* Used to iterate through segments */
   Fts5Buffer buf = {0,0,0};       /* Buffer used by fts5SegIterSeekInit() */
   Fts5StructureLevel *pLvl;
-  Fts5IndexIter *pNew;
+  Fts5Iter *pNew;
 
   assert( (pTerm==0 && nTerm==0) || iLevel<0 );
 
@@ -2917,16 +2934,16 @@ static void fts5MultiIterNew(
 }
 
 /*
-** Create an Fts5IndexIter that iterates through the doclist provided
+** Create an Fts5Iter that iterates through the doclist provided
 ** as the second argument.
 */
 static void fts5MultiIterNew2(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   Fts5Data *pData,                /* Doclist to iterate through */
   int bDesc,                      /* True for descending rowid order */
-  Fts5IndexIter **ppOut           /* New object */
+  Fts5Iter **ppOut                /* New object */
 ){
-  Fts5IndexIter *pNew;
+  Fts5Iter *pNew;
   pNew = fts5MultiIterAlloc(p, 2);
   if( pNew ){
     Fts5SegIter *pIter = &pNew->aSeg[1];
@@ -2961,7 +2978,7 @@ static void fts5MultiIterNew2(
 ** Return true if the iterator is at EOF or if an error has occurred. 
 ** False otherwise.
 */
-static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){
+static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){
   assert( p->rc 
       || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof 
   );
@@ -2973,7 +2990,7 @@ static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){
 ** to. If the iterator points to EOF when this function is called the
 ** results are undefined.
 */
-static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){
+static i64 fts5MultiIterRowid(Fts5Iter *pIter){
   assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
   return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
 }
@@ -2983,7 +3000,7 @@ static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){
 */
 static void fts5MultiIterNextFrom(
   Fts5Index *p, 
-  Fts5IndexIter *pIter, 
+  Fts5Iter *pIter, 
   i64 iMatch
 ){
   while( 1 ){
@@ -3000,7 +3017,7 @@ static void fts5MultiIterNextFrom(
 ** Return a pointer to a buffer containing the term associated with the 
 ** entry that the iterator currently points to.
 */
-static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){
+static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){
   Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
   *pn = p->term.n;
   return p->term.p;
@@ -3582,7 +3599,7 @@ static void fts5WriteInit(
 ** incremental merge operation. This function is called if the incremental
 ** merge step has finished but the input has not been completely exhausted.
 */
-static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
+static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
   int i;
   Fts5Buffer buf;
   memset(&buf, 0, sizeof(Fts5Buffer));
@@ -3660,7 +3677,7 @@ static void fts5IndexMergeLevel(
   Fts5Structure *pStruct = *ppStruct;
   Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
   Fts5StructureLevel *pLvlOut;
-  Fts5IndexIter *pIter = 0;       /* Iterator to read input data */
+  Fts5Iter *pIter = 0;       /* Iterator to read input data */
   int nRem = pnRem ? *pnRem : 0;  /* Output leaf pages left to write */
   int nInput;                     /* Number of input segments */
   Fts5SegWriter writer;           /* Writer object */
@@ -4342,7 +4359,7 @@ static int fts5IndexExtractCol(
 static int fts5AppendRowid(
   Fts5Index *p,
   i64 iDelta,
-  Fts5IndexIter *pMulti,
+  Fts5Iter *pMulti,
   Fts5Colset *pColset,
   Fts5Buffer *pBuf
 ){
@@ -4367,7 +4384,7 @@ static int fts5AppendRowid(
 static int fts5AppendPoslist(
   Fts5Index *p,
   i64 iDelta,
-  Fts5IndexIter *pMulti,
+  Fts5Iter *pMulti,
   Fts5Colset *pColset,
   Fts5Buffer *pBuf
 ){
@@ -4645,14 +4662,14 @@ static void fts5SetupPrefixIter(
   const u8 *pToken,               /* Buffer containing prefix to match */
   int nToken,                     /* Size of buffer pToken in bytes */
   Fts5Colset *pColset,            /* Restrict matches to these columns */
-  Fts5IndexIter **ppIter          /* OUT: New iterator */
+  Fts5Iter **ppIter          /* OUT: New iterator */
 ){
   Fts5Structure *pStruct;
   Fts5Buffer *aBuf;
   const int nBuf = 32;
 
   void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*);
-  int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*);
+  int (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Colset*, Fts5Buffer*);
   if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
     xMerge = fts5MergeRowidLists;
     xAppend = fts5AppendRowid;
@@ -4668,7 +4685,7 @@ static void fts5SetupPrefixIter(
     const int flags = FTS5INDEX_QUERY_SCAN;
     int i;
     i64 iLastRowid = 0;
-    Fts5IndexIter *p1 = 0;     /* Iterator used to gather data from index */
+    Fts5Iter *p1 = 0;     /* Iterator used to gather data from index */
     Fts5Data *pData;
     Fts5Buffer doclist;
     int bNewTerm = 1;
@@ -4932,6 +4949,154 @@ int sqlite3Fts5IndexWrite(
   return rc;
 }
 
+
+static int fts5IndexExtractColset (
+  Fts5Colset *pColset,            /* Colset to filter on */
+  const u8 *pPos, int nPos,       /* Position list */
+  Fts5Buffer *pBuf                /* Output buffer */
+){
+  int rc = SQLITE_OK;
+  int i;
+
+  fts5BufferZero(pBuf);
+  for(i=0; i<pColset->nCol; i++){
+    const u8 *pSub = pPos;
+    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
+    if( nSub ){
+      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
+    }
+  }
+  return rc;
+}
+
+/*
+** xSetOutputs callback used by detail=none tables.
+*/
+static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
+  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
+  pIter->base.iRowid = pSeg->iRowid;
+  pIter->base.nData = pSeg->nPos;
+}
+
+/*
+** xSetOutputs callback used by detail=full and detail=col tables when no
+** column filters are specified.
+*/
+static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
+  pIter->base.iRowid = pSeg->iRowid;
+  pIter->base.nData = pSeg->nPos;
+
+  assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE );
+  assert( pIter->pColset==0 || pIter->bFiltered );
+
+  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
+    /* All data is stored on the current page. Populate the output 
+    ** variables to point into the body of the page object. */
+    pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
+  }else{
+    /* The data is distributed over two or more pages. Copy it into the
+    ** Fts5Iter.poslist buffer and then set the output pointer to point
+    ** to this buffer.  */
+    fts5BufferZero(&pIter->poslist);
+    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
+    pIter->base.pData = pIter->poslist.p;
+  }
+}
+
+/*
+** xSetOutputs callback used by detail=col when there is a column filter.
+*/
+static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
+  Fts5Colset *pColset = pIter->pColset;
+  pIter->base.iRowid = pSeg->iRowid;
+
+  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
+  assert( pColset );
+
+  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && 0 ){
+    /* All data is stored on the current page. Populate the output 
+    ** variables to point into the body of the page object. */
+    Fts5PoslistWriter writer = {0};
+    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
+    int n = pSeg->nPos;
+    int iCol = 0;
+    i64 iPos = 0;
+    int iOff = 0;
+
+    fts5BufferZero(&pIter->poslist);
+    while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){
+      if( iPos==pColset->aiCol[iCol] ){
+        sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos);
+        if( ++iCol>=pColset->nCol ) break;
+      }
+    }
+
+  }else{
+    /* The data is distributed over two or more pages. Copy it into the
+    ** Fts5Iter.poslist buffer and then set the output pointer to point
+    ** to this buffer.  */
+    fts5BufferZero(&pIter->poslist);
+    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
+  }
+
+  pIter->base.pData = pIter->poslist.p;
+  pIter->base.nData = pIter->poslist.n;
+}
+
+/*
+** xSetOutputs callback used by detail=full when there is a column filter.
+*/
+static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
+  Fts5Colset *pColset = pIter->pColset;
+  pIter->base.iRowid = pSeg->iRowid;
+
+  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL );
+  assert( pColset );
+
+  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
+    /* All data is stored on the current page. Populate the output 
+    ** variables to point into the body of the page object. */
+    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
+    if( pColset->nCol==1 ){
+      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
+      pIter->base.pData = a;
+    }else{
+      fts5BufferZero(&pIter->poslist);
+      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
+      pIter->base.pData = pIter->poslist.p;
+      pIter->base.nData = pIter->poslist.n;
+    }
+  }else{
+    /* The data is distributed over two or more pages. Copy it into the
+    ** Fts5Iter.poslist buffer and then set the output pointer to point
+    ** to this buffer.  */
+    fts5BufferZero(&pIter->poslist);
+    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
+    pIter->base.pData = pIter->poslist.p;
+    pIter->base.nData = pIter->poslist.n;
+  }
+}
+
+static void fts5IterSetOutputCb(Fts5Iter *pIter){
+  int eDetail = pIter->pIndex->pConfig->eDetail;
+  if( eDetail==FTS5_DETAIL_NONE ){
+    pIter->xSetOutputs = fts5IterSetOutputs_None;
+  }
+
+  else if( pIter->pColset==0 || pIter->bFiltered ){
+    pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
+  }
+
+  else if( eDetail==FTS5_DETAIL_FULL ){
+    pIter->xSetOutputs = fts5IterSetOutputs_Full;
+  }
+
+  else{
+    assert( eDetail==FTS5_DETAIL_COLUMNS );
+    pIter->xSetOutputs = fts5IterSetOutputs_Col;
+  }
+}
+
 /*
 ** Open a new iterator to iterate though all rowid that match the 
 ** specified token or token prefix.
@@ -4944,22 +5109,27 @@ int sqlite3Fts5IndexQuery(
   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 ){
   Fts5Config *pConfig = p->pConfig;
-  Fts5IndexIter *pRet = 0;
-  int iIdx = 0;
+  Fts5Iter *pRet = 0;
   Fts5Buffer buf = {0, 0, 0};
 
   /* If the QUERY_SCAN flag is set, all other flags must be clear. */
   assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN );
 
   if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
+    int iIdx = 0;                 /* Index to search */
     memcpy(&buf.p[1], pToken, nToken);
 
-#ifdef SQLITE_DEBUG
-    /* If the QUERY_TEST_NOIDX flag was specified, then this must be a
+    /* Figure out which index to search and set iIdx accordingly. If this
+    ** is a prefix query for which there is no prefix index, set iIdx to
+    ** greater than pConfig->nPrefix to indicate that the query will be
+    ** satisfied by scanning multiple terms in the main index.
+    **
+    ** If the QUERY_TEST_NOIDX flag was specified, then this must be a
     ** prefix-query. Instead of using a prefix-index (if one exists), 
     ** evaluate the prefix query using the main FTS index. This is used
     ** for internal sanity checking by the integrity-check in debug 
     ** mode only.  */
+#ifdef SQLITE_DEBUG
     if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
       assert( flags & FTS5INDEX_QUERY_PREFIX );
       iIdx = 1+pConfig->nPrefix;
@@ -4973,6 +5143,7 @@ int sqlite3Fts5IndexQuery(
     }
 
     if( iIdx<=pConfig->nPrefix ){
+      /* Straight index lookup */
       Fts5Structure *pStruct = fts5StructureRead(p);
       buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx);
       if( pStruct ){
@@ -4980,17 +5151,25 @@ int sqlite3Fts5IndexQuery(
         fts5StructureRelease(pStruct);
       }
     }else{
+      /* Scan multiple terms in the main index */
       int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
       buf.p[0] = FTS5_MAIN_PREFIX;
       fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
     }
 
+    if( p->rc==SQLITE_OK ){
+      Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
+      pRet->pColset = pColset;
+      fts5IterSetOutputCb(pRet);
+      if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
+    }
     if( p->rc ){
-      sqlite3Fts5IterClose(pRet);
+      sqlite3Fts5IterClose(&pRet->base);
       pRet = 0;
       fts5CloseReader(p);
     }
-    *ppIter = pRet;
+
+    *ppIter = &pRet->base;
     sqlite3Fts5BufferFree(&buf);
   }
   return fts5IndexReturn(p);
@@ -5000,14 +5179,15 @@ int sqlite3Fts5IndexQuery(
 ** Return true if the iterator passed as the only argument is at EOF.
 */
 int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
-  assert( pIter->pIndex->rc==SQLITE_OK );
-  return pIter->bEof;
+  assert( ((Fts5Iter*)pIter)->pIndex->rc==SQLITE_OK );
+  return ((Fts5Iter*)pIter)->bEof;
 }
 
 /*
 ** Move to the next matching rowid. 
 */
-int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
+int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   assert( pIter->pIndex->rc==SQLITE_OK );
   fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
   return fts5IndexReturn(pIter->pIndex);
@@ -5016,7 +5196,8 @@ int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
 /*
 ** Move to the next matching term/rowid. Used by the fts5vocab module.
 */
-int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
+int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   Fts5Index *p = pIter->pIndex;
 
   assert( pIter->pIndex->rc==SQLITE_OK );
@@ -5039,7 +5220,8 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
 ** definition of "at or after" depends on whether this iterator iterates
 ** in ascending or descending rowid order.
 */
-int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
+int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
   return fts5IndexReturn(pIter->pIndex);
 }
@@ -5047,41 +5229,21 @@ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
 /*
 ** Return the current rowid.
 */
-i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
-  return fts5MultiIterRowid(pIter);
+i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIndexIter){
+  return fts5MultiIterRowid((Fts5Iter*)pIndexIter);
 }
 
 /*
 ** Return the current term.
 */
-const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){
+const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
   int n;
-  const char *z = (const char*)fts5MultiIterTerm(pIter, &n);
+  const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n);
   *pn = n-1;
   return &z[1];
 }
 
 
-static int fts5IndexExtractColset (
-  Fts5Colset *pColset,            /* Colset to filter on */
-  const u8 *pPos, int nPos,       /* Position list */
-  Fts5Buffer *pBuf                /* Output buffer */
-){
-  int rc = SQLITE_OK;
-  int i;
-
-  fts5BufferZero(pBuf);
-  for(i=0; i<pColset->nCol; i++){
-    const u8 *pSub = pPos;
-    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
-    if( nSub ){
-      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
-    }
-  }
-  return rc;
-}
-
-
 /*
 ** Return a pointer to a buffer containing a copy of the position list for
 ** the current entry. Output variable *pn is set to the size of the buffer 
@@ -5091,12 +5253,13 @@ static int fts5IndexExtractColset (
 ** field that starts the position list on disk.
 */
 int sqlite3Fts5IterPoslist(
-  Fts5IndexIter *pIter, 
+  Fts5IndexIter *pIndexIter, 
   Fts5Colset *pColset,            /* Column filter (or NULL) */
   const u8 **pp,                  /* OUT: Pointer to position-list data */
   int *pn,                        /* OUT: Size of position-list in bytes */
   i64 *piRowid                    /* OUT: Current rowid */
 ){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
   int eDetail = pIter->pIndex->pConfig->eDetail;
 
@@ -5133,10 +5296,11 @@ int sqlite3Fts5IterPoslist(
 }
 
 int sqlite3Fts5IterCollist(
-  Fts5IndexIter *pIter, 
+  Fts5IndexIter *pIndexIter, 
   const u8 **pp,                  /* OUT: Pointer to position-list data */
   int *pn                         /* OUT: Size of position-list in bytes */
 ){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
   *pp = pIter->poslist.p;
   *pn = pIter->poslist.n;
@@ -5148,7 +5312,8 @@ int sqlite3Fts5IterCollist(
 ** copies the position list into the buffer supplied as the second 
 ** argument.
 */
-int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
+int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIndexIter, Fts5Buffer *pBuf){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   Fts5Index *p = pIter->pIndex;
   Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
   assert( p->rc==SQLITE_OK );
@@ -5160,8 +5325,9 @@ int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
 /*
 ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
 */
-void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
-  if( pIter ){
+void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){
+  if( pIndexIter ){
+    Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
     Fts5Index *pIndex = pIter->pIndex;
     fts5MultiIterFree(pIter->pIndex, pIter);
     fts5CloseReader(pIndex);
@@ -5328,35 +5494,30 @@ static int fts5QueryCksum(
 ){
   int eDetail = p->pConfig->eDetail;
   u64 cksum = *pCksum;
-  Fts5IndexIter *pIdxIter = 0;
-  Fts5Buffer buf = {0, 0, 0};
-  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
+  Fts5IndexIter *pIter = 0;
+  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter);
 
-  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
-    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
+  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){
+    i64 rowid = sqlite3Fts5IterRowid(pIter);
 
     if( eDetail==FTS5_DETAIL_NONE ){
       cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
     }else{
-      rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
-      if( rc==SQLITE_OK ){
-        Fts5PoslistReader sReader;
-        for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
-            sReader.bEof==0;
-            sqlite3Fts5PoslistReaderNext(&sReader)
-        ){
-          int iCol = FTS5_POS2COLUMN(sReader.iPos);
-          int iOff = FTS5_POS2OFFSET(sReader.iPos);
-          cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
-        }
+      Fts5PoslistReader sReader;
+      for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader);
+          sReader.bEof==0;
+          sqlite3Fts5PoslistReaderNext(&sReader)
+      ){
+        int iCol = FTS5_POS2COLUMN(sReader.iPos);
+        int iOff = FTS5_POS2OFFSET(sReader.iPos);
+        cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
       }
     }
     if( rc==SQLITE_OK ){
-      rc = sqlite3Fts5IterNext(pIdxIter);
+      rc = sqlite3Fts5IterNext(pIter);
     }
   }
-  sqlite3Fts5IterClose(pIdxIter);
-  fts5BufferFree(&buf);
+  sqlite3Fts5IterClose(pIter);
 
   *pCksum = cksum;
   return rc;
@@ -5661,7 +5822,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   int eDetail = p->pConfig->eDetail;
   u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
   Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
-  Fts5IndexIter *pIter;           /* Used to iterate through entire index */
+  Fts5Iter *pIter;                /* Used to iterate through entire index */
   Fts5Structure *pStruct;         /* Index structure */
 
 #ifdef SQLITE_DEBUG
diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c
index ddc5576c5d..17aa5816c4 100644
--- a/ext/fts5/fts5_vocab.c
+++ b/ext/fts5/fts5_vocab.c
@@ -412,28 +412,29 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
         i64 iPos = 0;               /* 64-bit position read from poslist */
         int iOff = 0;               /* Current offset within position list */
 
+        pPos = pCsr->pIter->pData;
+        nPos = pCsr->pIter->nData;
         switch( pCsr->pConfig->eDetail ){
           case FTS5_DETAIL_FULL:
-            rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
-            if( rc==SQLITE_OK ){
-              if( pTab->eType==FTS5_VOCAB_ROW ){
-                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
-                  pCsr->aCnt[0]++;
-                }
-                pCsr->aDoc[0]++;
-              }else{
-                int iCol = -1;
-                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
-                  int ii = FTS5_POS2COLUMN(iPos);
-                  pCsr->aCnt[ii]++;
-                  if( iCol!=ii ){
-                    if( ii>=nCol ){
-                      rc = FTS5_CORRUPT;
-                      break;
-                    }
-                    pCsr->aDoc[ii]++;
-                    iCol = ii;
+            pPos = pCsr->pIter->pData;
+            nPos = pCsr->pIter->nData;
+            if( pTab->eType==FTS5_VOCAB_ROW ){
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+                pCsr->aCnt[0]++;
+              }
+              pCsr->aDoc[0]++;
+            }else{
+              int iCol = -1;
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+                int ii = FTS5_POS2COLUMN(iPos);
+                pCsr->aCnt[ii]++;
+                if( iCol!=ii ){
+                  if( ii>=nCol ){
+                    rc = FTS5_CORRUPT;
+                    break;
                   }
+                  pCsr->aDoc[ii]++;
+                  iCol = ii;
                 }
               }
             }
@@ -443,19 +444,14 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
             if( pTab->eType==FTS5_VOCAB_ROW ){
               pCsr->aDoc[0]++;
             }else{
-              Fts5Buffer buf = {0, 0, 0};
-              rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
-              if( rc==SQLITE_OK ){
-                while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
-                  assert_nc( iPos>=0 && iPos<nCol );
-                  if( iPos>=nCol ){
-                    rc = FTS5_CORRUPT;
-                    break;
-                  }
-                  pCsr->aDoc[iPos]++;
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
+                assert_nc( iPos>=0 && iPos<nCol );
+                if( iPos>=nCol ){
+                  rc = FTS5_CORRUPT;
+                  break;
                 }
+                pCsr->aDoc[iPos]++;
               }
-              sqlite3Fts5BufferFree(&buf);
             }
             break;
 
diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test
index 00b1328867..a5a522a466 100644
--- a/ext/fts5/test/fts5ac.test
+++ b/ext/fts5/test/fts5ac.test
@@ -250,7 +250,6 @@ foreach {tn2 sql} {
       FROM xx WHERE xx match $expr
     } $res
 
-
     set res [fts5_query_data $expr xx DESC]
     do_execsql_test 1.$tn2.$tn.[llength $res].desc {
       SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
diff --git a/manifest b/manifest
index 6c3814c3d9..99a9a8b139 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3TokenInit()\sutility\sfunction.
-D 2016-01-22T17:48:09.470
+C Experimental\sperformance\senhancements\sfor\sfts5.
+D 2016-01-22T19:48:34.825
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2
@@ -97,13 +97,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h 5599703af9c13512900a9f22fec39d48078d619d
+F ext/fts5/fts5Int.h f93aed3d05d1b3686c4dccf70544bcfc7aa117f1
 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
 F ext/fts5/fts5_buffer.c 7d3f6f01f8fdc45204e6a33925ef8478a67d28dd
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
-F ext/fts5/fts5_expr.c 4ab4504c54bbe24689c83411d8588f4ec99136e9
+F ext/fts5/fts5_expr.c 48b9131b74c8d3b8c12ba0f7995e2d60eecce9f2
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c 716c301835a122ba36910b4f821c87d26ae9a5d9
+F ext/fts5/fts5_index.c 19062d1f40ba6d88d786a986d1c20f9d741799d0
 F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3
 F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
@@ -112,13 +112,13 @@ F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
 F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8
 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
 F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
-F ext/fts5/fts5_vocab.c ee6df1a3be103414d7b7af833ae1885c7b83a9d0
+F ext/fts5/fts5_vocab.c f1b4308b9b7ec8e659d0c9b39ddc8f1aeee47a1a
 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 6d0d74b695c4be055a8ba1dd807f22a2abc95b5e
 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
-F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced
+F ext/fts5/test/fts5ac.test dec95549e007dd9be52aa435cdcd0f08e14e64d0
 F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
@@ -1419,7 +1419,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e60461e984b8df09256bb0d733dbfae52568a145
-R 03ab6310cefadc63b5c7c7da966c2348
-U drh
-Z 0c42ba520d71deaa3e0a21b23ff67f9d
+P 7323175337b7ba85ac932ca892b28860f6a5b688
+R bc65520844365f59e43048a22901b688
+T *branch * fts5-perf
+T *sym-fts5-perf *
+T -sym-trunk *
+U dan
+Z 20f25a74d681d80924173ee060583d9e
diff --git a/manifest.uuid b/manifest.uuid
index e9fd60917b..fc3a1227d6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7323175337b7ba85ac932ca892b28860f6a5b688
\ No newline at end of file
+b5a57b812fd6a734cf2a342bf0b730ae18912d73
\ No newline at end of file

From 8498f387be1e02e4053c4bf5dcf4ded8a7103484 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 23 Jan 2016 14:45:36 +0000
Subject: [PATCH 2/6] Fix a problem involving detail=col and column filters.

FossilOrigin-Name: 7558a0ad2276e91f2faced8ea405d9fdb4fa0c6e
---
 ext/fts5/fts5_index.c | 14 ++++++++++----
 manifest              | 15 ++++++---------
 manifest.uuid         |  2 +-
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 2ea57b04f9..8323842dc6 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -5013,21 +5013,26 @@ static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
   assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
   assert( pColset );
 
-  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && 0 ){
+  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
     /* All data is stored on the current page. Populate the output 
     ** variables to point into the body of the page object. */
     Fts5PoslistWriter writer = {0};
     const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
     int n = pSeg->nPos;
     int iCol = 0;
+    int iCVal = pColset->aiCol[0];
     i64 iPos = 0;
     int iOff = 0;
 
     fts5BufferZero(&pIter->poslist);
     while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){
-      if( iPos==pColset->aiCol[iCol] ){
-        sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos);
-        if( ++iCol>=pColset->nCol ) break;
+      while( iPos>=iCVal ){
+        if( iPos==iCVal ){
+          sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos);
+        }
+        if( ++iCol>=pColset->nCol ) goto setoutputs_col_out;
+        assert( pColset->aiCol[iCol]>iCVal );
+        iCVal = pColset->aiCol[iCol];
       }
     }
 
@@ -5039,6 +5044,7 @@ static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
     fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
   }
 
+setoutputs_col_out:
   pIter->base.pData = pIter->poslist.p;
   pIter->base.nData = pIter->poslist.n;
 }
diff --git a/manifest b/manifest
index 99a9a8b139..0168850d28 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Experimental\sperformance\senhancements\sfor\sfts5.
-D 2016-01-22T19:48:34.825
+C Fix\sa\sproblem\sinvolving\sdetail=col\sand\scolumn\sfilters.
+D 2016-01-23T14:45:36.706
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2
@@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c 7d3f6f01f8fdc45204e6a33925ef8478a67d28dd
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
 F ext/fts5/fts5_expr.c 48b9131b74c8d3b8c12ba0f7995e2d60eecce9f2
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c 19062d1f40ba6d88d786a986d1c20f9d741799d0
+F ext/fts5/fts5_index.c bd5476edd4d6ef37fc389794a78f47f29f521634
 F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3
 F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
@@ -1419,10 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 7323175337b7ba85ac932ca892b28860f6a5b688
-R bc65520844365f59e43048a22901b688
-T *branch * fts5-perf
-T *sym-fts5-perf *
-T -sym-trunk *
+P b5a57b812fd6a734cf2a342bf0b730ae18912d73
+R 20bdd3799e5ba562672d44f9df7f8c2a
 U dan
-Z 20f25a74d681d80924173ee060583d9e
+Z fc938fd5a5718db0c9c10bacac344b80
diff --git a/manifest.uuid b/manifest.uuid
index fc3a1227d6..7a9fdf27eb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b5a57b812fd6a734cf2a342bf0b730ae18912d73
\ No newline at end of file
+7558a0ad2276e91f2faced8ea405d9fdb4fa0c6e
\ No newline at end of file

From e9eb1593f54e9c5e0490dbea5ad0c6a0dc11ef4d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 23 Jan 2016 18:51:59 +0000
Subject: [PATCH 3/6] Fix an fts5 problem with using both xPhraseFirst() and
 xPhraseFirstColumn() within a single statement in detail=col mode.

FossilOrigin-Name: 72d53699bf0dcdb9d2a22e229989d7435f061399
---
 ext/fts5/fts5Int.h              |  6 +--
 ext/fts5/fts5_aux.c             |  2 +-
 ext/fts5/fts5_buffer.c          |  4 +-
 ext/fts5/fts5_expr.c            | 79 +++++++++++----------------------
 ext/fts5/fts5_index.c           | 79 ---------------------------------
 ext/fts5/fts5_main.c            |  4 +-
 ext/fts5/fts5_storage.c         |  2 +-
 ext/fts5/fts5_tokenize.c        |  2 +-
 ext/fts5/fts5_vocab.c           |  3 +-
 ext/fts5/test/fts5_common.tcl   |  3 +-
 ext/fts5/test/fts5synonym2.test | 37 ++++++++++++++-
 manifest                        | 32 ++++++-------
 manifest.uuid                   |  2 +-
 13 files changed, 90 insertions(+), 165 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index b8486dfdfb..48ab980921 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -29,7 +29,7 @@ typedef unsigned short u16;
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
-#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
+#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
 
 #define testcase(x)
 #define ALWAYS(x) 1
@@ -388,8 +388,6 @@ int sqlite3Fts5IterEof(Fts5IndexIter*);
 int sqlite3Fts5IterNext(Fts5IndexIter*);
 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
-int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*);
-int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf);
 
 /*
 ** Close an iterator opened by sqlite3Fts5IndexQuery().
@@ -402,8 +400,6 @@ void sqlite3Fts5IterClose(Fts5IndexIter*);
 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
 
-int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);
-
 
 /*
 ** Insert or remove data to or from the index. Each time a document is 
diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index 011064d405..79bed1c8f7 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -544,7 +544,7 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){
   int rc = SQLITE_OK;             /* Return code */
   int i;                          /* To iterate through builtin functions */
 
-  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
+  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
     rc = pApi->xCreateFunction(pApi,
         aBuiltin[i].zFunc,
         aBuiltin[i].pUserData,
diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index 9bd4fddb82..d69adf7d24 100644
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -322,7 +322,7 @@ int sqlite3Fts5TermsetAdd(
   *pbPresent = 0;
   if( p ){
     int i;
-    int hash = 13;
+    u32 hash = 13;
     Fts5TermsetEntry *pEntry;
 
     /* Calculate a hash value for this term. This is the same hash checksum
@@ -339,7 +339,7 @@ int sqlite3Fts5TermsetAdd(
       if( pEntry->iIdx==iIdx 
           && pEntry->nTerm==nTerm 
           && memcmp(pEntry->pTerm, pTerm, nTerm)==0 
-        ){
+      ){
         *pbPresent = 1;
         break;
       }
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 290baf4e2c..3a1497f05d 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -306,7 +306,7 @@ static int fts5ExprSynonymList(
   int bCollist, 
   Fts5Colset *pColset,
   i64 iRowid,
-  int *pbDel,                     /* OUT: Caller should sqlite3_free(*pa) */
+  Fts5Buffer *pBuf,               /* Use this buffer for space if required */
   u8 **pa, int *pn
 ){
   Fts5PoslistReader aStatic[4];
@@ -320,22 +320,7 @@ static int fts5ExprSynonymList(
   for(p=pTerm; p; p=p->pSynonym){
     Fts5IndexIter *pIter = p->pIter;
     if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
-      const u8 *a;
-      int n;
-
-      if( 0 && bCollist ){
-        rc = sqlite3Fts5IterCollist(pIter, &a, &n);
-      }else{
-        i64 dummy;
-#if 0
-        rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
-#endif
-        a = pIter->pData;
-        n = pIter->nData;
-      }
-
-      if( rc!=SQLITE_OK ) goto synonym_poslist_out;
-      if( n==0 ) continue;
+      if( pIter->nData==0 ) continue;
       if( nIter==nAlloc ){
         int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
         Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
@@ -348,20 +333,19 @@ static int fts5ExprSynonymList(
         if( aIter!=aStatic ) sqlite3_free(aIter);
         aIter = aNew;
       }
-      sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]);
+      sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]);
       assert( aIter[nIter].bEof==0 );
       nIter++;
     }
   }
 
-  assert( *pbDel==0 );
   if( nIter==1 ){
     *pa = (u8*)aIter[0].a;
     *pn = aIter[0].n;
   }else{
     Fts5PoslistWriter writer = {0};
-    Fts5Buffer buf = {0,0,0};
     i64 iPrev = -1;
+    fts5BufferZero(pBuf);
     while( 1 ){
       int i;
       i64 iMin = FTS5_LARGEST_INT64;
@@ -376,15 +360,12 @@ static int fts5ExprSynonymList(
         }
       }
       if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break;
-      rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin);
+      rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin);
       iPrev = iMin;
     }
-    if( rc ){
-      sqlite3_free(buf.p);
-    }else{
-      *pa = buf.p;
-      *pn = buf.n;
-      *pbDel = 1;
+    if( rc==SQLITE_OK ){
+      *pa = pBuf->p;
+      *pn = pBuf->n;
     }
   }
 
@@ -421,7 +402,7 @@ static int fts5ExprPhraseIsMatch(
 
   /* If the aStatic[] array is not large enough, allocate a large array
   ** using sqlite3_malloc(). This approach could be improved upon. */
-  if( pPhrase->nTerm>(int)ArraySize(aStatic) ){
+  if( pPhrase->nTerm>ArraySize(aStatic) ){
     int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
     aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
     if( !aIter ) return SQLITE_NOMEM;
@@ -431,23 +412,23 @@ static int fts5ExprPhraseIsMatch(
   /* Initialize a term iterator for each term in the phrase */
   for(i=0; i<pPhrase->nTerm; i++){
     Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
-    i64 dummy;
     int n = 0;
     int bFlag = 0;
-    const u8 *a = 0;
+    u8 *a = 0;
     if( pTerm->pSynonym ){
+      Fts5Buffer buf = {0, 0, 0};
       rc = fts5ExprSynonymList(
-          pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
+          pTerm, 0, pColset, pNode->iRowid, &buf, &a, &n
       );
+      if( rc ){
+        sqlite3_free(a);
+        goto ismatch_out;
+      }
+      if( a==buf.p ) bFlag = 1;
     }else{
-      Fts5IndexIter *pIter = pTerm->pIter;
-#if 0
-      rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
-#endif
-      a = pIter->pData;
-      n = pIter->nData;
+      a = (u8*)pTerm->pIter->pData;
+      n = pTerm->pIter->nData;
     }
-    if( rc!=SQLITE_OK ) goto ismatch_out;
     sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
     aIter[i].bFlag = (u8)bFlag;
     if( aIter[i].bEof ) goto ismatch_out;
@@ -562,7 +543,7 @@ static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){
 
   /* If the aStatic[] array is not large enough, allocate a large array
   ** using sqlite3_malloc(). This approach could be improved upon. */
-  if( pNear->nPhrase>(int)ArraySize(aStatic) ){
+  if( pNear->nPhrase>ArraySize(aStatic) ){
     int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
     a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
   }else{
@@ -1387,10 +1368,10 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
       sqlite3_free(pTerm->zTerm);
       sqlite3Fts5IterClose(pTerm->pIter);
-
       for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
         pNext = pSyn->pSynonym;
         sqlite3Fts5IterClose(pSyn->pIter);
+        fts5BufferFree((Fts5Buffer*)&pSyn[1]);
         sqlite3_free(pSyn);
       }
     }
@@ -1478,13 +1459,13 @@ static int fts5ParseTokenize(
   assert( pPhrase==0 || pPhrase->nTerm>0 );
   if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
     Fts5ExprTerm *pSyn;
-    int nByte = sizeof(Fts5ExprTerm) + nToken+1;
+    int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
     pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
     if( pSyn==0 ){
       rc = SQLITE_NOMEM;
     }else{
       memset(pSyn, 0, nByte);
-      pSyn->zTerm = (char*)&pSyn[1];
+      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
       memcpy(pSyn->zTerm, pToken, nToken);
       pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
       pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
@@ -2263,7 +2244,7 @@ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
   int rc = SQLITE_OK;
   void *pCtx = (void*)pGlobal;
 
-  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aFunc); i++){
+  for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){
     struct Fts5ExprFunc *p = &aFunc[i];
     rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
   }
@@ -2509,18 +2490,10 @@ int sqlite3Fts5ExprPhraseCollist(
   ){
     Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
     if( pTerm->pSynonym ){
-      int bDel = 0;
-      u8 *a;
+      Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1];
       rc = fts5ExprSynonymList(
-          pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist
+          pTerm, 1, 0, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
       );
-      if( bDel ){
-        sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a);
-        *ppCollist = pPhrase->poslist.p;
-        sqlite3_free(a);
-      }else{
-        *ppCollist = a;
-      }
     }else{
       *ppCollist = pPhrase->aTerm[0].pIter->pData;
       *pnCollist = pPhrase->aTerm[0].pIter->nData;
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 8323842dc6..e147199ef8 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -5249,85 +5249,6 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
   return &z[1];
 }
 
-
-/*
-** Return a pointer to a buffer containing a copy of the position list for
-** the current entry. Output variable *pn is set to the size of the buffer 
-** in bytes before returning.
-**
-** The returned position list does not include the "number of bytes" varint
-** field that starts the position list on disk.
-*/
-int sqlite3Fts5IterPoslist(
-  Fts5IndexIter *pIndexIter, 
-  Fts5Colset *pColset,            /* Column filter (or NULL) */
-  const u8 **pp,                  /* OUT: Pointer to position-list data */
-  int *pn,                        /* OUT: Size of position-list in bytes */
-  i64 *piRowid                    /* OUT: Current rowid */
-){
-  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
-  int eDetail = pIter->pIndex->pConfig->eDetail;
-
-  assert( pIter->pIndex->rc==SQLITE_OK );
-  *piRowid = pSeg->iRowid;
-  if( eDetail==FTS5_DETAIL_NONE ){
-    *pn = pSeg->nPos;
-  }else
-  if( eDetail==FTS5_DETAIL_FULL 
-   && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
-  ){
-    u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
-    if( pColset==0 || pIter->bFiltered ){
-      *pn = pSeg->nPos;
-      *pp = pPos;
-    }else if( pColset->nCol==1 ){
-      *pp = pPos;
-      *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]);
-    }else{
-      fts5BufferZero(&pIter->poslist);
-      fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist);
-      *pp = pIter->poslist.p;
-      *pn = pIter->poslist.n;
-    }
-  }else{
-    fts5BufferZero(&pIter->poslist);
-    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
-    if( eDetail==FTS5_DETAIL_FULL ){
-      *pp = pIter->poslist.p;
-    }
-    *pn = pIter->poslist.n;
-  }
-  return fts5IndexReturn(pIter->pIndex);
-}
-
-int sqlite3Fts5IterCollist(
-  Fts5IndexIter *pIndexIter, 
-  const u8 **pp,                  /* OUT: Pointer to position-list data */
-  int *pn                         /* OUT: Size of position-list in bytes */
-){
-  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
-  *pp = pIter->poslist.p;
-  *pn = pIter->poslist.n;
-  return SQLITE_OK;
-}
-
-/*
-** This function is similar to sqlite3Fts5IterPoslist(), except that it
-** copies the position list into the buffer supplied as the second 
-** argument.
-*/
-int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIndexIter, Fts5Buffer *pBuf){
-  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  Fts5Index *p = pIter->pIndex;
-  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
-  assert( p->rc==SQLITE_OK );
-  fts5BufferZero(pBuf);
-  fts5SegiterPoslist(p, pSeg, 0, pBuf);
-  return fts5IndexReturn(p);
-}
-
 /*
 ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
 */
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 7447f9b5e1..ec045da6bf 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -538,7 +538,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   for(i=0; i<pInfo->nConstraint; i++){
     struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
     int j;
-    for(j=0; j<(int)ArraySize(aConstraint); j++){
+    for(j=0; j<ArraySize(aConstraint); j++){
       struct Constraint *pC = &aConstraint[j];
       if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
         if( p->usable ){
@@ -585,7 +585,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
 
   /* Assign argvIndex values to each constraint in use. */
   iNext = 1;
-  for(i=0; i<(int)ArraySize(aConstraint); i++){
+  for(i=0; i<ArraySize(aConstraint); i++){
     struct Constraint *pC = &aConstraint[i];
     if( pC->iConsIndex>=0 ){
       pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++;
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 56383619d1..da9524dea3 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -338,7 +338,7 @@ int sqlite3Fts5StorageClose(Fts5Storage *p){
     int i;
 
     /* Finalize all SQL statements */
-    for(i=0; i<(int)ArraySize(p->aStmt); i++){
+    for(i=0; i<ArraySize(p->aStmt); i++){
       sqlite3_finalize(p->aStmt[i]);
     }
 
diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c
index e60183c095..790adaae47 100644
--- a/ext/fts5/fts5_tokenize.c
+++ b/ext/fts5/fts5_tokenize.c
@@ -1220,7 +1220,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
   int rc = SQLITE_OK;             /* Return code */
   int i;                          /* To iterate through builtin functions */
 
-  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
+  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
     rc = pApi->xCreateTokenizer(pApi,
         aBuiltin[i].zName,
         (void*)pApi,
diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c
index 17aa5816c4..b17e8974fb 100644
--- a/ext/fts5/fts5_vocab.c
+++ b/ext/fts5/fts5_vocab.c
@@ -184,7 +184,7 @@ static int fts5VocabInitVtab(
     
     rc = fts5VocabTableType(zType, pzErr, &eType);
     if( rc==SQLITE_OK ){
-      assert( eType>=0 && eType<sizeof(azSchema)/sizeof(azSchema[0]) );
+      assert( eType>=0 && eType<ArraySize(azSchema) );
       rc = sqlite3_declare_vtab(db, azSchema[eType]);
     }
 
@@ -407,7 +407,6 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 
       assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
       while( rc==SQLITE_OK ){
-        i64 dummy;
         const u8 *pPos; int nPos;   /* Position list */
         i64 iPos = 0;               /* 64-bit position read from poslist */
         int iOff = 0;               /* Current offset within position list */
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index e7c61af00c..cf688dcc0d 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -48,7 +48,8 @@ proc fts5_test_poslist2 {cmd} {
     }
   }
 
-  set res
+  #set res
+  sort_poslist $res
 }
 
 proc fts5_test_collist {cmd} {
diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test
index dddaa17af8..7e92822c33 100644
--- a/ext/fts5/test/fts5synonym2.test
+++ b/ext/fts5/test/fts5synonym2.test
@@ -27,6 +27,21 @@ foreach_detail_mode $testprefix {
 fts5_tclnum_register db
 fts5_aux_test_functions db
 
+proc fts5_test_bothlist {cmd} {
+
+  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
+    set bFirst 1
+    $cmd xPhraseColumnForeach $i c { 
+      lappend CL $i.$c 
+      if {$bFirst} { $cmd xPhraseForeach $i c o { lappend PL $i.$c.$o } }
+      set bFirst 0
+    }
+  }
+
+  list [sort_poslist $PL] $CL
+}
+sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist
+
 proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
 sqlite3_fts5_create_function db fts5_rowid fts5_rowid
 
@@ -89,6 +104,8 @@ do_execsql_test 1.$tok.0.2 {
 }
 
 foreach {tn expr} {
+  2.1 "one OR two OR three OR four"
+
   1.1 "one"   1.2 "two"   1.3 "three"   1.4 "four"
   1.5 "v"     1.6 "vi"    1.7 "vii"     1.8 "viii"
   1.9 "9"    1.10 "0"    1.11 "1"      1.12 "2"
@@ -113,13 +130,31 @@ foreach {tn expr} {
 
   set res [fts5_query_data $expr ss ASC ::tclnum_syn]
   do_execsql_test 1.$tok.$tn.[llength $res].asc.1 {
-    SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
+    SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr)
   } $res
 
   do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
     SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
+  } $res
+
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
+    SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr)
     ORDER BY rank ASC
   } $res
+
+  set res2 [list]
+  foreach {a b c} $res { lappend res2 $a $c $b }
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.3 {
+    SELECT rowid, fts5_test_collist(ss), fts5_test_poslist2(ss) FROM ss($expr)
+  } $res2
+
+  set res3 [list]
+  foreach {a b c} $res { lappend res3 $a [list $b $c] }
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.3 {
+    SELECT rowid, fts5_test_bothlist(ss) FROM ss($expr)
+  } $res3
+
+
 }
 
 }
diff --git a/manifest b/manifest
index e3a6b78f3f..fbcb4d2d1b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\strunk\schanges\s(including\sfixes\sfor\swarnings\sin\sfts5)\swith\sthis\sbranch.
-D 2016-01-23T16:20:16.389
+C Fix\san\sfts5\sproblem\swith\susing\sboth\sxPhraseFirst()\sand\sxPhraseFirstColumn()\swithin\sa\ssingle\sstatement\sin\sdetail=col\smode.
+D 2016-01-23T18:51:59.865
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9
@@ -97,25 +97,25 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h a810589cf2fedd7dcfc9ba7871d2a5a09e850ace
-F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
-F ext/fts5/fts5_buffer.c 0b8e1f84fec3ec01f7e17f8d4f17c46856b31e10
+F ext/fts5/fts5Int.h 6e0f90eb4872654a5b98130dec16965716525c9a
+F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d
+F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
-F ext/fts5/fts5_expr.c 48b9131b74c8d3b8c12ba0f7995e2d60eecce9f2
+F ext/fts5/fts5_expr.c a66b9694519d9c336d9bdbd46ea22e7e14aef412
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c bd5476edd4d6ef37fc389794a78f47f29f521634
-F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3
-F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e
+F ext/fts5/fts5_index.c 722d8717e3167dd05fa48af970352932052da318
+F ext/fts5/fts5_main.c 3886bbfc5ac1d9df29979823ddf2b68241e1127e
+F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
 F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
-F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8
+F ext/fts5/fts5_tokenize.c 4d5c4f183c7d07d144bc219b92da1ea0e962fae3
 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
 F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
-F ext/fts5/fts5_vocab.c f1b4308b9b7ec8e659d0c9b39ddc8f1aeee47a1a
+F ext/fts5/fts5_vocab.c 3ef401a8d6932db56368de32f446eb9fe73aa623
 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl 6d0d74b695c4be055a8ba1dd807f22a2abc95b5e
+F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9
 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test dec95549e007dd9be52aa435cdcd0f08e14e64d0
@@ -177,7 +177,7 @@ F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46
 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
-F ext/fts5/test/fts5synonym2.test eadb00c73ef0653258873e756b7e9102e0687539
+F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e
 F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
@@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 7558a0ad2276e91f2faced8ea405d9fdb4fa0c6e 3be336aa893f9eb0837d7d66c83bab1489792b9a
-R 27b07c29574081bf2a6f1649bb5e2393
+P ceccc9ad788fb4da9120915741995b9f088f85ff
+R e7d7753020b6f70b109a92a8ac037ba7
 U dan
-Z 165b70c16fc218fdaf7b93acd8863bd2
+Z 648645363fb131dd10e5d7321a173b9d
diff --git a/manifest.uuid b/manifest.uuid
index 3e212691ce..1891619c18 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ceccc9ad788fb4da9120915741995b9f088f85ff
\ No newline at end of file
+72d53699bf0dcdb9d2a22e229989d7435f061399
\ No newline at end of file

From fee981e5b97a79b26308c2dc3f946264e4ee8028 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 26 Jan 2016 17:08:22 +0000
Subject: [PATCH 4/6] Enhance fts5txt2db.tcl, a script used to generate
 fts5/fts4 databases for performance testing.

FossilOrigin-Name: c646e40350e5aa91abcf52de61fb31275bad38f9
---
 ext/fts5/tool/fts5txt2db.tcl | 200 +++++++++++++++++++++++++----------
 manifest                     |  12 +--
 manifest.uuid                |   2 +-
 3 files changed, 152 insertions(+), 62 deletions(-)

diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl
index c22c5dae72..c6a58e6edd 100644
--- a/ext/fts5/tool/fts5txt2db.tcl
+++ b/ext/fts5/tool/fts5txt2db.tcl
@@ -1,13 +1,127 @@
 
 
-proc usage {} {
-  puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..."
+#-------------------------------------------------------------------------
+# Command line options processor.
+#
+proc command_line_error {O E {msg ""}} {
+  if {$msg != ""} {
+    puts stderr "Error: $msg"
+    puts stderr ""
+  }
+
+  set L [list]
+  foreach o $O {
+    if {[llength $o]==1} {
+      lappend L [string toupper $o]
+    }
+  }
+
+  puts stderr "Usage: $::argv0 ?SWITCHES? $L"
   puts stderr ""
-  puts stderr "Options are"
-  puts stderr "  -fts5"
-  puts stderr "  -fts4"
-  puts stderr "  -colsize <list of column sizes>"
-  puts stderr {
+  puts stderr "Switches are:"
+  foreach o $O {
+    if {[llength $o]==3} {
+      foreach {a b c} $o {}
+      puts stderr [format "    -%-15s %s (default \"%s\")" "$a VAL" $c $b]
+    } elseif {[llength $o]==2} {
+      foreach {a b} $o {}
+      puts stderr [format "    -%-15s %s" $a $b]
+    }
+  }
+  puts stderr ""
+  puts stderr $E
+  exit -1
+}
+
+proc process_command_line {avar lArgs O E} {
+
+  upvar $avar A
+  set zTrailing ""       ;# True if ... is present in $O
+  set lPosargs [list]
+
+  # Populate A() with default values. Also, for each switch in the command
+  # line spec, set an entry in the idx() array as follows:
+  #
+  #  {tblname t1 "table name to use"}  
+  #      -> [set idx(-tblname) {tblname t1 "table name to use"}  
+  #
+  # For each position parameter, append its name to $lPosargs. If the ...
+  # specifier is present, set $zTrailing to the name of the prefix.
+  #
+  foreach o $O {
+    set nm [lindex $o 0]
+    set nArg [llength $o]
+    switch -- $nArg {
+      1 {
+        if {[string range $nm end-2 end]=="..."} {
+          set zTrailing [string range $nm 0 end-3]
+        } else {
+          lappend lPosargs $nm
+        }
+      }
+      2 {
+        set A($nm) 0
+        set idx(-$nm) $o
+      }
+      3 {
+        set A($nm) [lindex $o 1]
+        set idx(-$nm) $o
+      }
+      default {
+        error "Error in command line specification"
+      }
+    }
+  }
+
+  # Set explicitly specified option values
+  #
+  set nArg [llength $lArgs]
+  for {set i 0} {$i < $nArg} {incr i} {
+    set opt [lindex $lArgs $i]
+    if {[string range $opt 0 0]!="-" || $opt=="--"} break
+    set c [array names idx "${opt}*"]
+    if {[llength $c]==0} { command_line_error $O $E "Unrecognized option: $opt"}
+    if {[llength $c]>1}  { command_line_error $O $E "Ambiguous option: $opt"}
+
+    if {[llength $idx($c)]==3} {
+      if {$i==[llength $lArgs]-1} {
+        command_line_error $O $E "Option requires argument: $c" 
+      }
+      incr i
+      set A([lindex $idx($c) 0]) [lindex $lArgs $i]
+    } else {
+      set A([lindex $idx($c) 0]) 1
+    }
+  }
+
+  # Deal with position arguments.
+  #
+  set nPosarg [llength $lPosargs]
+  set nRem [expr $nArg - $i]
+  if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} {
+    command_line_error $O $E
+  }
+  for {set j 0} {$j < $nPosarg} {incr j} {
+    set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]]
+  }
+  if {$zTrailing!=""} {
+    set A($zTrailing) [lrange $lArgs [expr $j+$i] end]
+  }
+}
+# End of command line options processor.
+#-------------------------------------------------------------------------
+
+
+process_command_line A $argv {
+  {fts5                 "use fts5"}
+  {fts4                 "use fts4"}
+  {colsize   "10 10 10" "list of column sizes"}
+  {tblname   "t1"       "table name to create"}
+  {detail    "full"     "Fts5 detail mode to use"}
+  {repeat    1          "Load each file this many times"}
+  database
+  file...
+} {
 This script is designed to create fts4/5 tables with more than one column.
 The -colsize option should be set to a Tcl list of integer values, one for
 each column in the table. Each value is the number of tokens that will be
@@ -22,59 +136,27 @@ of the -colsize list. The next N2 are used for the second column of the first
 row, and so on. Rows are added to the table until the entire list of tokens
 is exhausted.
 }
-  exit -1
+
+if {$A(fts4)} {
+  set A(fts) fts4
+} else {
+  set A(fts) fts5
 }
 
-set O(aColSize)       [list 10 10 10]
-set O(tblname)        t1
-set O(fts)            fts5
-
-
-set options_with_values {-colsize}
-
-for {set i 0} {$i < [llength $argv]} {incr i} {
-  set opt [lindex $argv $i]
-  if {[string range $opt 0 0]!="-"} break
-
-  if {[lsearch $options_with_values $opt]>=0} {
-    incr i
-    if {$i==[llength $argv]} usage
-    set val [lindex $argv $i]
-  }
-
-  switch -- $opt {
-    -colsize {
-      set O(aColSize) $val
-    }
-
-    -fts4 {
-      set O(fts) fts4
-    }
-
-    -fts5 {
-      set O(fts) fts5
-    }
-  }
-}
-
-if {$i > [llength $argv]-2} usage
-set O(db) [lindex $argv $i]
-set O(files) [lrange $argv [expr $i+1] end]
-
-sqlite3 db $O(db)
+sqlite3 db $A(database)
 
 # Create the FTS table in the db. Return a list of the table columns.
 #
 proc create_table {} {
-  global O
+  global A
   set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
 
-  set nCol [llength $O(aColSize)]
+  set nCol [llength $A(colsize)]
   set cols [lrange $cols 0 [expr $nCol-1]]
 
-  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) ("
+  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $A(tblname) USING $A(fts) ("
   append sql [join $cols ,]
-  append sql ");"
+  if {$A(fts)=="fts5"} { append sql ",detail=$A(detail));" }
 
   db eval $sql
   return $cols
@@ -89,27 +171,35 @@ proc readfile {file} {
   split $data
 }
 
+proc repeat {L n} {
+  set res [list]
+  for {set i 0} {$i < $n} {incr i} {
+    set res [concat $res $L]
+  }
+  set res
+}
+
 
 # Load all the data into a big list of tokens.
 #
 set tokens [list]
-foreach f $O(files) {
-  set tokens [concat $tokens [readfile $f]]
+foreach f $A(file) {
+  set tokens [concat $tokens [repeat [readfile $f] $A(repeat)]]
 }
 
 set N [llength $tokens]
 set i 0
 set cols [create_table]
-set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]"
+set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])"
 foreach c [lrange $cols 1 end] {
-  append sql ", \$A($c)"
+  append sql ", \$R($c)"
 }
 append sql ")"
 
 db eval BEGIN
   while {$i < $N} {
-    foreach c $cols s $O(aColSize) {
-      set A($c) [lrange $tokens $i [expr $i+$s-1]]
+    foreach c $cols s $A(colsize) {
+      set R($c) [lrange $tokens $i [expr $i+$s-1]]
       incr i $s
     }
     db eval $sql
diff --git a/manifest b/manifest
index fbcb4d2d1b..32c610bbaf 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sfts5\sproblem\swith\susing\sboth\sxPhraseFirst()\sand\sxPhraseFirstColumn()\swithin\sa\ssingle\sstatement\sin\sdetail=col\smode.
-D 2016-01-23T18:51:59.865
+C Enhance\sfts5txt2db.tcl,\sa\sscript\sused\sto\sgenerate\sfts5/fts4\sdatabases\sfor\sperformance\stesting.
+D 2016-01-26T17:08:22.193
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9
@@ -189,7 +189,7 @@ F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
 F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477
 F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221
-F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84
+F ext/fts5/tool/fts5txt2db.tcl ae308338b2da1646dea456ab66706acdde8c714e
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
@@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ceccc9ad788fb4da9120915741995b9f088f85ff
-R e7d7753020b6f70b109a92a8ac037ba7
+P 72d53699bf0dcdb9d2a22e229989d7435f061399
+R e572f7330a1bdca73a606e620f2735b9
 U dan
-Z 648645363fb131dd10e5d7321a173b9d
+Z 9f71d918f9bd09bf4fa550347b1ed369
diff --git a/manifest.uuid b/manifest.uuid
index 1891619c18..0db14bcc12 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-72d53699bf0dcdb9d2a22e229989d7435f061399
\ No newline at end of file
+c646e40350e5aa91abcf52de61fb31275bad38f9
\ No newline at end of file

From 1c16bccb79888f775b4e1fbdca00d8f272afed17 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 26 Jan 2016 19:30:49 +0000
Subject: [PATCH 5/6] Improve the performance of fts5 column filters on
 detail=col tables.

FossilOrigin-Name: 249a2d070c34bf884a04cb248b9691e239f2871c
---
 ext/fts5/fts5_index.c          | 68 ++++++++++++++++++++++++++++++----
 ext/fts5/test/fts5ac.test      |  2 +-
 ext/fts5/test/fts5simple3.test | 44 ++++++++++++++++++++++
 ext/fts5/tool/fts5speed.tcl    |  2 +
 manifest                       | 17 +++++----
 manifest.uuid                  |  2 +-
 6 files changed, 117 insertions(+), 18 deletions(-)
 create mode 100644 ext/fts5/test/fts5simple3.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e147199ef8..6d442de799 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -5003,6 +5003,53 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
   }
 }
 
+/*
+** xSetOutputs callback used when: detail=col when there is a column filter.
+**
+**   * detail=col,
+**   * there is a column filter, and
+**   * the table contains 32 or fewer columns.
+*/
+static void fts5IterSetOutputs_Col32(Fts5Iter *pIter, Fts5SegIter *pSeg){
+  Fts5Colset *pColset = pIter->pColset;
+  pIter->base.iRowid = pSeg->iRowid;
+
+  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
+  assert( pColset );
+
+  fts5BufferZero(&pIter->poslist);
+  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
+    int i;
+    int iPrev = 0;
+    u32 m = 0;
+    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
+    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
+
+    while( a<pEnd ){
+      iPrev += (int)a[0] - 2;
+      m |= (1 << iPrev);
+      a++;
+    }
+
+    iPrev = 0;
+    a = pIter->poslist.p;
+    for(i=0; i<pColset->nCol; i++){
+      int iCol = pColset->aiCol[i];
+      if( m & (1 << iCol) ){
+        *a++ = (iCol - iPrev) + 2;
+        iPrev = iCol;
+      }
+    }
+    pIter->poslist.n = a - pIter->poslist.p;
+
+  }else{
+    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
+  }
+
+  pIter->base.pData = pIter->poslist.p;
+  pIter->base.nData = pIter->poslist.n;
+}
+
 /*
 ** xSetOutputs callback used by detail=col when there is a column filter.
 */
@@ -5083,9 +5130,9 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
   }
 }
 
-static void fts5IterSetOutputCb(Fts5Iter *pIter){
-  int eDetail = pIter->pIndex->pConfig->eDetail;
-  if( eDetail==FTS5_DETAIL_NONE ){
+static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
+  Fts5Config *pConfig = pIter->pIndex->pConfig;
+  if( pConfig->eDetail==FTS5_DETAIL_NONE ){
     pIter->xSetOutputs = fts5IterSetOutputs_None;
   }
 
@@ -5093,13 +5140,18 @@ static void fts5IterSetOutputCb(Fts5Iter *pIter){
     pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
   }
 
-  else if( eDetail==FTS5_DETAIL_FULL ){
+  else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
     pIter->xSetOutputs = fts5IterSetOutputs_Full;
   }
 
   else{
-    assert( eDetail==FTS5_DETAIL_COLUMNS );
-    pIter->xSetOutputs = fts5IterSetOutputs_Col;
+    assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
+    if( pConfig->nCol<=32 ){
+      pIter->xSetOutputs = fts5IterSetOutputs_Col32;
+      sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
+    }else{
+      pIter->xSetOutputs = fts5IterSetOutputs_Col;
+    }
   }
 }
 
@@ -5166,8 +5218,8 @@ int sqlite3Fts5IndexQuery(
     if( p->rc==SQLITE_OK ){
       Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
       pRet->pColset = pColset;
-      fts5IterSetOutputCb(pRet);
-      if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
+      fts5IterSetOutputCb(&p->rc, pRet);
+      if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
     }
     if( p->rc ){
       sqlite3Fts5IterClose(&pRet->base);
diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test
index a5a522a466..61b3230772 100644
--- a/ext/fts5/test/fts5ac.test
+++ b/ext/fts5/test/fts5ac.test
@@ -158,8 +158,8 @@ foreach {tn2 sql} {
   #-------------------------------------------------------------------------
   #
   foreach {tn expr} {
-    1.2 "a   OR b"
     1.1 "a   AND b"
+    1.2 "a   OR b"
     1.3 "o"
     1.4 "b q"
     1.5 "e a e"
diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test
new file mode 100644
index 0000000000..b84a3e56dd
--- /dev/null
+++ b/ext/fts5/test/fts5simple3.test
@@ -0,0 +1,44 @@
+# 2015 September 05
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5simple3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+fts5_aux_test_functions db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
+  INSERT INTO t1 VALUES('a', 'b', 'c');
+  INSERT INTO t1 VALUES('x', 'x', 'x');
+}
+
+do_execsql_test 1.1 {
+  SELECT rowid, fts5_test_collist(t1) FROM t1('a:a');
+} {1 0.0}
+
+do_execsql_test 1.2 {
+  SELECT rowid, fts5_test_collist(t1) FROM t1('b:x');
+} {2 0.1}
+
+do_execsql_test 1.3 {
+  SELECT rowid, fts5_test_collist(t1) FROM t1('b:a');
+} {}
+
+
+finish_test
+
diff --git a/ext/fts5/tool/fts5speed.tcl b/ext/fts5/tool/fts5speed.tcl
index 1b060ea759..8e1c7f3899 100644
--- a/ext/fts5/tool/fts5speed.tcl
+++ b/ext/fts5/tool/fts5speed.tcl
@@ -11,6 +11,8 @@ set Q {
   {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"}
   {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"}
   {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"}
+
+  {2   "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:the'"}
 }
 
 proc usage {} {
diff --git a/manifest b/manifest
index 32c610bbaf..2e90cb8536 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sfts5txt2db.tcl,\sa\sscript\sused\sto\sgenerate\sfts5/fts4\sdatabases\sfor\sperformance\stesting.
-D 2016-01-26T17:08:22.193
+C Improve\sthe\sperformance\sof\sfts5\scolumn\sfilters\son\sdetail=col\stables.
+D 2016-01-26T19:30:49.768
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9
@@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
 F ext/fts5/fts5_expr.c a66b9694519d9c336d9bdbd46ea22e7e14aef412
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c 722d8717e3167dd05fa48af970352932052da318
+F ext/fts5/fts5_index.c b34b7257f73929dc1b25c420aad2453dcbe36128
 F ext/fts5/fts5_main.c 3886bbfc5ac1d9df29979823ddf2b68241e1127e
 F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
@@ -118,7 +118,7 @@ F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9
 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
-F ext/fts5/test/fts5ac.test dec95549e007dd9be52aa435cdcd0f08e14e64d0
+F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
@@ -176,6 +176,7 @@ F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46
+F ext/fts5/test/fts5simple3.test e671b36bc4dbd4f5095e66cb04473cba9f680f53
 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
 F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e
 F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
@@ -188,7 +189,7 @@ F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
 F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477
-F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221
+F ext/fts5/tool/fts5speed.tcl 47f0031e6ac564964f4f4805e439ea665e848df2
 F ext/fts5/tool/fts5txt2db.tcl ae308338b2da1646dea456ab66706acdde8c714e
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
@@ -1419,7 +1420,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 72d53699bf0dcdb9d2a22e229989d7435f061399
-R e572f7330a1bdca73a606e620f2735b9
+P c646e40350e5aa91abcf52de61fb31275bad38f9
+R a44979856d96ea38951595ad6e15598a
 U dan
-Z 9f71d918f9bd09bf4fa550347b1ed369
+Z a6d46df0ea079f8f36131bee8e2cf5b7
diff --git a/manifest.uuid b/manifest.uuid
index 0db14bcc12..54ccb7acb1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c646e40350e5aa91abcf52de61fb31275bad38f9
\ No newline at end of file
+249a2d070c34bf884a04cb248b9691e239f2871c
\ No newline at end of file

From 18205c781d4bb15d5a26801844212f231a4aed9d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 26 Jan 2016 20:08:50 +0000
Subject: [PATCH 6/6] Further minor performance improvements and code-size
 reductions related to fts5 column filters on detail=col tables.

FossilOrigin-Name: b4ac61aeee976296e7719949cd4fb496147a29e8
---
 ext/fts5/fts5_index.c | 119 +++++++++++++++---------------------------
 manifest              |  12 ++---
 manifest.uuid         |   2 +-
 3 files changed, 50 insertions(+), 83 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 6d442de799..ca3feb86f7 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -5004,96 +5004,63 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
 }
 
 /*
-** xSetOutputs callback used when: detail=col when there is a column filter.
-**
-**   * detail=col,
-**   * there is a column filter, and
-**   * the table contains 32 or fewer columns.
+** xSetOutputs callback used by detail=col when there is a column filter
+** and there are 100 or more columns. Also called as a fallback from
+** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
 */
-static void fts5IterSetOutputs_Col32(Fts5Iter *pIter, Fts5SegIter *pSeg){
-  Fts5Colset *pColset = pIter->pColset;
-  pIter->base.iRowid = pSeg->iRowid;
-
-  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
-  assert( pColset );
-
+static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
   fts5BufferZero(&pIter->poslist);
-  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
-    int i;
-    int iPrev = 0;
-    u32 m = 0;
-    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
-    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
-
-    while( a<pEnd ){
-      iPrev += (int)a[0] - 2;
-      m |= (1 << iPrev);
-      a++;
-    }
-
-    iPrev = 0;
-    a = pIter->poslist.p;
-    for(i=0; i<pColset->nCol; i++){
-      int iCol = pColset->aiCol[i];
-      if( m & (1 << iCol) ){
-        *a++ = (iCol - iPrev) + 2;
-        iPrev = iCol;
-      }
-    }
-    pIter->poslist.n = a - pIter->poslist.p;
-
-  }else{
-    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
-  }
-
+  fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);
+  pIter->base.iRowid = pSeg->iRowid;
   pIter->base.pData = pIter->poslist.p;
   pIter->base.nData = pIter->poslist.n;
 }
 
 /*
-** xSetOutputs callback used by detail=col when there is a column filter.
+** xSetOutputs callback used when: 
+**
+**   * detail=col,
+**   * there is a column filter, and
+**   * the table contains 100 or fewer columns. 
+**
+** The last point is to ensure all column numbers are stored as 
+** single-byte varints.
 */
-static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
-  Fts5Colset *pColset = pIter->pColset;
-  pIter->base.iRowid = pSeg->iRowid;
+static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){
 
   assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
-  assert( pColset );
+  assert( pIter->pColset );
 
-  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
-    /* All data is stored on the current page. Populate the output 
-    ** variables to point into the body of the page object. */
-    Fts5PoslistWriter writer = {0};
-    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
-    int n = pSeg->nPos;
-    int iCol = 0;
-    int iCVal = pColset->aiCol[0];
-    i64 iPos = 0;
-    int iOff = 0;
+  if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){
+    fts5IterSetOutputs_Col(pIter, pSeg);
+  }else{
+    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
+    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
+    int iPrev = 0;
+    int *aiCol = pIter->pColset->aiCol;
+    int *aiColEnd = &aiCol[pIter->pColset->nCol];
 
-    fts5BufferZero(&pIter->poslist);
-    while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){
-      while( iPos>=iCVal ){
-        if( iPos==iCVal ){
-          sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos);
-        }
-        if( ++iCol>=pColset->nCol ) goto setoutputs_col_out;
-        assert( pColset->aiCol[iCol]>iCVal );
-        iCVal = pColset->aiCol[iCol];
+    u8 *aOut = pIter->poslist.p;
+    int iPrevOut = 0;
+
+    pIter->base.iRowid = pSeg->iRowid;
+
+    while( a<pEnd ){
+      iPrev += (int)a++[0] - 2;
+      while( *aiCol<iPrev ){
+        aiCol++;
+        if( aiCol==aiColEnd ) goto setoutputs_col_out;
+      }
+      if( *aiCol==iPrev ){
+        *aOut++ = (iPrev - iPrevOut) + 2;
+        iPrevOut = iPrev;
       }
     }
 
-  }else{
-    /* The data is distributed over two or more pages. Copy it into the
-    ** Fts5Iter.poslist buffer and then set the output pointer to point
-    ** to this buffer.  */
-    fts5BufferZero(&pIter->poslist);
-    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
-  }
-
 setoutputs_col_out:
-  pIter->base.pData = pIter->poslist.p;
-  pIter->base.nData = pIter->poslist.n;
+    pIter->base.pData = pIter->poslist.p;
+    pIter->base.nData = aOut - pIter->poslist.p;
+  }
 }
 
 /*
@@ -5146,8 +5113,8 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
 
   else{
     assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
-    if( pConfig->nCol<=32 ){
-      pIter->xSetOutputs = fts5IterSetOutputs_Col32;
+    if( pConfig->nCol<=100 ){
+      pIter->xSetOutputs = fts5IterSetOutputs_Col100;
       sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
     }else{
       pIter->xSetOutputs = fts5IterSetOutputs_Col;
diff --git a/manifest b/manifest
index 2e90cb8536..173bd3a03a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\sperformance\sof\sfts5\scolumn\sfilters\son\sdetail=col\stables.
-D 2016-01-26T19:30:49.768
+C Further\sminor\sperformance\simprovements\sand\scode-size\sreductions\srelated\sto\sfts5\scolumn\sfilters\son\sdetail=col\stables.
+D 2016-01-26T20:08:50.348
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9
@@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
 F ext/fts5/fts5_expr.c a66b9694519d9c336d9bdbd46ea22e7e14aef412
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c b34b7257f73929dc1b25c420aad2453dcbe36128
+F ext/fts5/fts5_index.c 5558bfbeaf364cc67f937e25753ceed8757cb6d1
 F ext/fts5/fts5_main.c 3886bbfc5ac1d9df29979823ddf2b68241e1127e
 F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
@@ -1420,7 +1420,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c646e40350e5aa91abcf52de61fb31275bad38f9
-R a44979856d96ea38951595ad6e15598a
+P 249a2d070c34bf884a04cb248b9691e239f2871c
+R a28f5f4bedd812396ab85ff0aefd35c4
 U dan
-Z a6d46df0ea079f8f36131bee8e2cf5b7
+Z 56fb7a3fddaab518f0c4eb551a124662
diff --git a/manifest.uuid b/manifest.uuid
index 54ccb7acb1..708359e9ec 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-249a2d070c34bf884a04cb248b9691e239f2871c
\ No newline at end of file
+b4ac61aeee976296e7719949cd4fb496147a29e8
\ No newline at end of file