Try to reuse sorter statements in fts5. Does not work due to circular references on VTable object.

FossilOrigin-Name: bc14e64bdffb165fc5468339a7eaa5f6707537bf

ext/fts3/fts3.h

@@ -20,6 +20,7 @@ extern "C" {
 #endif  /* __cplusplus */
 
 int sqlite3Fts3Init(sqlite3 *db);
+int sqlite3Fts5Init(sqlite3 *db);
 
 #ifdef __cplusplus
 }  /* extern "C" */

ext/fts5/fts5.h

@@ -0,0 +1,184 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Interfaces to extend FTS5. Using the interfaces defined in this file, 
+** FTS5 may be extended with:
+**
+**     * custom tokenizers, and
+**     * custom auxiliary functions.
+*/
+
+
+#ifndef _FTS5_H
+#define _FTS5_H
+
+#include "sqlite3.h"
+
+/*************************************************************************
+** CUSTOM AUXILIARY FUNCTIONS
+**
+** Virtual table implemenations may overload SQL functions by implementing
+** the sqlite3_module.xFindFunction() method.
+*/
+
+typedef struct Fts5ExtensionApi Fts5ExtensionApi;
+typedef struct Fts5Context Fts5Context;
+
+typedef void (*fts5_extension_function)(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  sqlite3_context *pCtx,          /* Context for returning result/error */
+  int nVal,                       /* Number of values in apVal[] array */
+  sqlite3_value **apVal           /* Array of trailing arguments */
+);
+
+/*
+**
+** xUserData(pFts):
+**
+**   Return a copy of the context pointer the extension function was 
+**   registered with.
+**
+**
+** xColumnTotalSize(pFts, iCol, pnToken):
+**
+**   Returns the total number of tokens in column iCol, considering all
+**   rows in the FTS5 table.
+**
+**
+** xColumnCount:
+**   Returns the number of columns in the FTS5 table.
+**
+** xColumnSize:
+**   Reports the size in tokens of a column value from the current row.
+**
+** xColumnText:
+**   Reports the size in tokens of a column value from the current row.
+**
+** xPhraseCount:
+**   Returns the number of phrases in the current query expression.
+**
+** xPhraseSize:
+**   Returns the number of tokens in phrase iPhrase of the query. Phrases
+**   are numbered starting from zero.
+**
+** xRowid:
+**   Returns the rowid of the current row.
+**
+** xPoslist:
+**   Iterate through instances of phrase iPhrase in the current row. 
+**
+**   At EOF, a non-zero value is returned and output variable iPos set to -1.
+**
+** xTokenize:
+**   Tokenize text using the tokenizer belonging to the FTS5 table.
+**
+**
+** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
+**
+**   This API function is used to query the FTS table for phrase iPhrase
+**   of the current query. Specifically, a query equivalent to:
+**
+**       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC
+**
+**   with $p set to a phrase equivalent to the phrase iPhrase of the
+**   current query is executed. For each row visited, the callback function
+**   passed as the fourth argument is invoked. The context and API objects 
+**   passed to the callback function may be used to access the properties of
+**   each matched row. Invoking Api.xUserData() returns a copy of the pointer
+**   passed as the third argument to pUserData.
+**
+**   If the callback function returns any value other than SQLITE_OK, the
+**   query is abandoned and the xQueryPhrase function returns immediately.
+**   If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
+**   Otherwise, the error code is propagated upwards.
+**
+**   If the query runs to completion without incident, SQLITE_OK is returned.
+**   Or, if some error occurs before the query completes or is aborted by
+**   the callback, an SQLite error code is returned.
+**
+**
+** xSetAuxdata(pFts5, pAux, xDelete)
+**
+**   Save the pointer passed as the second argument as the extension functions 
+**   "auxiliary data". The pointer may then be retrieved by the current or any
+**   future invocation of the same fts5 extension function made as part of
+**   of the same MATCH query using the xGetAuxdata() API.
+**
+**   Each extension function is allocated a single auxiliary data slot per
+**   query. If the extension function is invoked more than once by the SQL
+**   query, then all invocations share a single auxiliary data context.
+**
+**   If there is already an auxiliary data pointer when this function is
+**   invoked, then it is replaced by the new pointer. If an xDelete callback
+**   was specified along with the original pointer, it is invoked at this
+**   point.
+**
+**   The xDelete callback, if one is specified, is also invoked on the
+**   auxiliary data pointer after the FTS5 query has finished.
+**
+**
+** xGetAuxdata(pFts5, bClear)
+**
+**   Returns the current auxiliary data pointer for the fts5 extension 
+**   function. See the xSetAuxdata() method for details.
+**
+**   If the bClear argument is non-zero, then the auxiliary data is cleared
+**   (set to NULL) before this function returns. In this case the xDelete,
+**   if any, is not invoked.
+**
+**
+** xRowCount(pFts5, pnRow)
+**
+**   This function is used to retrieve the total number of rows in the table.
+**   In other words, the same value that would be returned by:
+**
+**        SELECT count(*) FROM ftstable;
+*/
+struct Fts5ExtensionApi {
+  int iVersion;                   /* Currently always set to 1 */
+
+  void *(*xUserData)(Fts5Context*);
+
+  int (*xColumnCount)(Fts5Context*);
+  int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
+  int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
+
+  int (*xTokenize)(Fts5Context*, 
+    const char *pText, int nText, /* Text to tokenize */
+    void *pCtx,                   /* Context passed to xToken() */
+    int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
+  );
+
+  int (*xPhraseCount)(Fts5Context*);
+  int (*xPhraseSize)(Fts5Context*, int iPhrase);
+
+  sqlite3_int64 (*xRowid)(Fts5Context*);
+  int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
+  int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
+  int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, sqlite3_int64 *piPos);
+
+  int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
+    int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
+  );
+  int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
+  void *(*xGetAuxdata)(Fts5Context*, int bClear);
+};
+
+#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
+#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
+
+/* 
+** CUSTOM AUXILIARY FUNCTIONS
+*************************************************************************/
+#endif /* _FTS5_H */
+

ext/fts5/fts5Int.h

@@ -0,0 +1,444 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+#ifndef _FTS5INT_H
+#define _FTS5INT_H
+
+#include "fts5.h"
+#include "sqliteInt.h"
+#include "fts3_tokenizer.h"
+
+
+/*
+** Maximum number of prefix indexes on single FTS5 table. This must be
+** less than 32. If it is set to anything large than that, an #error
+** directive in fts5_index.c will cause the build to fail.
+*/
+#define FTS5_MAX_PREFIX_INDEXES 31
+
+#define FTS5_DEFAULT_NEARDIST 10
+
+/* Name of rank column */
+#define FTS5_RANK_NAME "rank"
+
+/**************************************************************************
+** Interface to code in fts5_config.c. fts5_config.c contains contains code
+** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
+*/
+
+typedef struct Fts5Config Fts5Config;
+
+/*
+** An instance of the following structure encodes all information that can
+** be gleaned from the CREATE VIRTUAL TABLE statement.
+*/
+struct Fts5Config {
+  sqlite3 *db;                    /* Database handle */
+  char *zDb;                      /* Database holding FTS index (e.g. "main") */
+  char *zName;                    /* Name of FTS index */
+  int nCol;                       /* Number of columns */
+  char **azCol;                   /* Column names */
+  int nPrefix;                    /* Number of prefix indexes */
+  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
+  sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance for this table */
+};
+
+int sqlite3Fts5ConfigParse(sqlite3*, int, const char**, Fts5Config**, char**);
+void sqlite3Fts5ConfigFree(Fts5Config*);
+
+int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
+
+int sqlite3Fts5Tokenize(
+  Fts5Config *pConfig,            /* FTS5 Configuration object */
+  const char *pText, int nText,   /* Text to tokenize */
+  void *pCtx,                     /* Context passed to xToken() */
+  int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
+);
+
+void sqlite3Fts5Dequote(char *z);
+
+/*
+** End of interface to code in fts5_config.c.
+**************************************************************************/
+
+/**************************************************************************
+*/
+
+/*
+** Buffer object for the incremental building of string data.
+*/
+typedef struct Fts5Buffer Fts5Buffer;
+struct Fts5Buffer {
+  u8 *p;
+  int n;
+  int nSpace;
+};
+
+int sqlite3Fts5BufferGrow(int*, Fts5Buffer*, int);
+void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
+void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
+void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
+void sqlite3Fts5BufferFree(Fts5Buffer*);
+void sqlite3Fts5BufferZero(Fts5Buffer*);
+void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
+void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
+void sqlite3Fts5BufferAppendListElem(int*, Fts5Buffer*, const char*, int);
+
+#define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
+#define fts5BufferGrow(a,b,c)         sqlite3Fts5BufferGrow(a,b,c)
+#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
+#define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
+#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
+#define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
+
+typedef struct Fts5PoslistReader Fts5PoslistReader;
+struct Fts5PoslistReader {
+  /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
+  int iCol;                       /* If (iCol>=0), this column only */
+  const u8 *a;                    /* Position list to iterate through */
+  int n;                          /* Size of buffer at a[] in bytes */
+  int i;                          /* Current offset in a[] */
+
+  /* Output variables */
+  int bEof;                       /* Set to true at EOF */
+  i64 iPos;                       /* (iCol<<32) + iPos */
+};
+int sqlite3Fts5PoslistReaderInit(
+  int iCol,                       /* If (iCol>=0), this column only */
+  const u8 *a, int n,             /* Poslist buffer to iterate through */
+  Fts5PoslistReader *pIter        /* Iterator object to initialize */
+);
+int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
+
+typedef struct Fts5PoslistWriter Fts5PoslistWriter;
+struct Fts5PoslistWriter {
+  i64 iPrev;
+};
+int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
+
+int sqlite3Fts5PoslistNext(
+  const u8 *a, int n,             /* Buffer containing poslist */
+  int *pi,                        /* IN/OUT: Offset within a[] */
+  int *piCol,                     /* IN/OUT: Current column */
+  int *piOff                      /* IN/OUT: Current token offset */
+);
+
+int sqlite3Fts5PoslistNext64(
+  const u8 *a, int n,             /* Buffer containing poslist */
+  int *pi,                        /* IN/OUT: Offset within a[] */
+  i64 *piOff                      /* IN/OUT: Current offset */
+);
+
+/*
+** End of interface to code in fts5_buffer.c.
+**************************************************************************/
+
+/**************************************************************************
+** Interface to code in fts5_index.c. fts5_index.c contains contains code
+** to access the data stored in the %_data table.
+*/
+
+typedef struct Fts5Index Fts5Index;
+typedef struct Fts5IndexIter Fts5IndexIter;
+
+/*
+** Values used as part of the flags argument passed to IndexQuery().
+*/
+#define FTS5INDEX_QUERY_PREFIX 0x0001       /* Prefix query */
+#define FTS5INDEX_QUERY_ASC    0x0002       /* Docs in ascending rowid order */
+#define FTS5INDEX_QUERY_MATCH  0x0004       /* Use the iMatch arg to Next() */
+
+/*
+** Create/destroy an Fts5Index object.
+*/
+int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
+int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy);
+
+/*
+** for(
+**   pIter = sqlite3Fts5IndexQuery(p, "token", 5, 0);
+**   0==sqlite3Fts5IterEof(pIter);
+**   sqlite3Fts5IterNext(pIter)
+** ){
+**   i64 iDocid = sqlite3Fts5IndexDocid(pIter);
+** }
+*/
+
+/*
+** Open a new iterator to iterate though all docids that match the 
+** specified token or token prefix.
+*/
+Fts5IndexIter *sqlite3Fts5IndexQuery(
+  Fts5Index *p,                   /* FTS index to query */
+  const char *pToken, int nToken, /* Token (or prefix) to query for */
+  int flags                       /* Mask of FTS5INDEX_QUERY_X flags */
+);
+
+/*
+** Docid list iteration.
+*/
+int  sqlite3Fts5IterEof(Fts5IndexIter*);
+void sqlite3Fts5IterNext(Fts5IndexIter*, i64 iMatch);
+i64  sqlite3Fts5IterRowid(Fts5IndexIter*);
+
+/*
+** Obtain the position list that corresponds to the current position.
+*/
+const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter*, int *pn);
+
+/*
+** Close an iterator opened by sqlite3Fts5IndexQuery().
+*/
+void sqlite3Fts5IterClose(Fts5IndexIter*);
+
+/*
+** Insert or remove data to or from the index. Each time a document is 
+** added to or removed from the index, this function is called one or more
+** times.
+**
+** For an insert, it must be called once for each token in the new document.
+** If the operation is a delete, it must be called (at least) once for each
+** unique token in the document with an iCol value less than zero. The iPos
+** argument is ignored for a delete.
+*/
+void sqlite3Fts5IndexWrite(
+  Fts5Index *p,                   /* Index to write to */
+  int iCol,                       /* Column token appears in (-ve -> delete) */
+  int iPos,                       /* Position of token within column */
+  const char *pToken, int nToken  /* Token to add or remove to or from index */
+);
+
+/*
+** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
+** document iDocid.
+*/
+void sqlite3Fts5IndexBeginWrite(
+  Fts5Index *p,                   /* Index to write to */
+  i64 iDocid                      /* Docid to add or remove data from */
+);
+
+/*
+** Flush any data stored in the in-memory hash tables to the database.
+**
+** This is called whenever (a) the main transaction is committed or (b) a 
+** new sub-transaction is opened.
+*/
+void sqlite3Fts5IndexFlush(Fts5Index *p);
+
+int sqlite3Fts5IndexSync(Fts5Index *p);
+
+/*
+** Discard any data stored in the in-memory hash tables. Do not write it
+** to the database. Additionally, assume that the contents of the %_data
+** table may have changed on disk. So any in-memory caches of %_data 
+** records must be invalidated.
+**
+** This is called (a) whenever a main or sub-transaction is rolled back, 
+** and (b) whenever the read transaction is closed.
+*/
+int sqlite3Fts5IndexRollback(Fts5Index *p);
+
+/*
+** Retrieve and clear the current error code, respectively.
+*/
+int sqlite3Fts5IndexErrcode(Fts5Index*);
+void sqlite3Fts5IndexReset(Fts5Index*);
+
+/*
+** Get (bSet==0) or set (bSet!=0) the "averages" record.
+*/
+void sqlite3Fts5IndexAverages(Fts5Index *p, int bSet, int nAvg, int *aAvg);
+
+/*
+** Functions called by the storage module as part of integrity-check.
+*/
+u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int);
+int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
+
+/* Called during startup to register a UDF with SQLite */
+int sqlite3Fts5IndexInit(sqlite3*);
+
+void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
+
+int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf);
+int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
+
+/*
+** End of interface to code in fts5_index.c.
+**************************************************************************/
+
+/**************************************************************************
+** Interface to code in fts5_storage.c. fts5_storage.c contains contains 
+** code to access the data stored in the %_content and %_docsize tables.
+*/
+
+#define FTS5_STMT_SCAN_ASC   0    /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
+#define FTS5_STMT_SCAN_DESC  1    /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
+#define FTS5_STMT_LOOKUP     2    /* SELECT rowid, * FROM ... WHERE rowid=? */
+#define FTS5_STMT_SORTER_DESC 3   /* SELECT ... ORDER BY rank ASC */
+#define FTS5_STMT_SORTER_ASC  4   /* SELECT ... ORDER BY rank ASC */
+
+typedef struct Fts5Storage Fts5Storage;
+
+int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
+int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy);
+
+int sqlite3Fts5DropTable(Fts5Config*, const char *zPost);
+int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, char **pzErr);
+
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
+int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
+
+int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
+
+int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
+void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
+
+int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
+int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
+int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
+
+
+/*
+** End of interface to code in fts5_storage.c.
+**************************************************************************/
+
+
+/**************************************************************************
+** Interface to code in fts5_expr.c. 
+*/
+typedef struct Fts5Expr Fts5Expr;
+typedef struct Fts5ExprNode Fts5ExprNode;
+typedef struct Fts5Parse Fts5Parse;
+typedef struct Fts5Token Fts5Token;
+typedef struct Fts5ExprPhrase Fts5ExprPhrase;
+typedef struct Fts5ExprNearset Fts5ExprNearset;
+
+struct Fts5Token {
+  const char *p;                  /* Token text (not NULL terminated) */
+  int n;                          /* Size of buffer p in bytes */
+};
+
+/* Parse a MATCH expression. */
+int sqlite3Fts5ExprNew(
+  Fts5Config *pConfig, 
+  const char *zExpr,
+  Fts5Expr **ppNew, 
+  char **pzErr
+);
+
+/*
+** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bAsc);
+**     rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
+**     rc = sqlite3Fts5ExprNext(pExpr)
+** ){
+**   // The document with rowid iRowid matches the expression!
+**   i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
+** }
+*/
+int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, int bAsc);
+int sqlite3Fts5ExprNext(Fts5Expr*);
+int sqlite3Fts5ExprEof(Fts5Expr*);
+i64 sqlite3Fts5ExprRowid(Fts5Expr*);
+
+void sqlite3Fts5ExprFree(Fts5Expr*);
+
+/* Called during startup to register a UDF with SQLite */
+int sqlite3Fts5ExprInit(sqlite3*);
+
+int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
+int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
+int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
+
+int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
+
+/*******************************************
+** The fts5_expr.c API above this point is used by the other hand-written
+** C code in this module. The interfaces below this point are called by
+** the parser code in fts5parse.y.  */
+
+void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
+
+Fts5ExprNode *sqlite3Fts5ParseNode(
+  Fts5Parse *pParse,
+  int eType,
+  Fts5ExprNode *pLeft,
+  Fts5ExprNode *pRight,
+  Fts5ExprNearset *pNear
+);
+
+Fts5ExprPhrase *sqlite3Fts5ParseTerm(
+  Fts5Parse *pParse, 
+  Fts5ExprPhrase *pPhrase, 
+  Fts5Token *pToken,
+  int bPrefix
+);
+
+Fts5ExprNearset *sqlite3Fts5ParseNearset(
+  Fts5Parse*, 
+  Fts5ExprNearset*,
+  Fts5ExprPhrase* 
+);
+
+void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
+void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
+void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
+
+void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
+void sqlite3Fts5ParseSetColumn(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
+void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
+void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
+
+/*
+** End of interface to code in fts5_expr.c.
+**************************************************************************/
+
+
+/**************************************************************************
+** Interface to code in fts5.c. 
+*/
+typedef struct Fts5Global Fts5Global;
+
+int sqlite3Fts5CreateAux(
+    Fts5Global*, 
+    const char*, 
+    void*, 
+    fts5_extension_function, 
+    void(*)(void*)
+);
+/*
+** End of interface to code in fts5.c.
+**************************************************************************/
+
+
+/**************************************************************************
+** Interface to code in fts5_aux.c. 
+*/
+
+int sqlite3Fts5AuxInit(Fts5Global*);
+/*
+** End of interface to code in fts5_aux.c.
+**************************************************************************/
+
+/**************************************************************************
+** Interface to code in fts5_sorter.c. 
+*/
+typedef struct Fts5Sorter Fts5Sorter;
+
+int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp);
+
+/*
+** End of interface to code in fts5_sorter.c.
+**************************************************************************/
+
+#endif

ext/fts5/fts5_aux.c

@@ -0,0 +1,980 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+*/
+
+#include "fts5Int.h"
+#include <math.h>
+
+typedef struct SnipPhrase SnipPhrase;
+typedef struct SnipIter SnipIter;
+typedef struct SnippetCtx SnippetCtx;
+
+struct SnipPhrase {
+  u64 mask;                       /* Current mask */
+  int nToken;                     /* Tokens in this phrase */
+  int i;                          /* Current offset in phrase poslist */
+  i64 iPos;                       /* Next position in phrase (-ve -> EOF) */
+};
+
+struct SnipIter {
+  i64 iLast;                      /* Last token position of current snippet */
+  int nScore;                     /* Score of current snippet */
+
+  const Fts5ExtensionApi *pApi;
+  Fts5Context *pFts;
+  u64 szmask;                     /* Mask used to on SnipPhrase.mask */
+  int nPhrase;                    /* Number of phrases */
+  SnipPhrase aPhrase[0];       /* Array of size nPhrase */
+};
+
+struct SnippetCtx {
+  int iFirst;                     /* Offset of first token to record */
+  int nToken;                     /* Size of aiStart[] and aiEnd[] arrays */
+  int iSeen;                      /* Set to largest offset seen */
+  int *aiStart; 
+  int *aiEnd;
+};
+
+static int fts5SnippetCallback(
+  void *pContext,                 /* Pointer to Fts5Buffer object */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd,                       /* End offset of token */
+  int iPos                        /* Position offset of token */
+){
+  int rc = SQLITE_OK;
+  SnippetCtx *pCtx = (SnippetCtx*)pContext;
+  int iOff = iPos - pCtx->iFirst;
+
+  if( iOff>=0 ){
+    if( iOff < pCtx->nToken ){
+      pCtx->aiStart[iOff] = iStart;
+      pCtx->aiEnd[iOff] = iEnd;
+    }
+    pCtx->iSeen = iPos;
+    if( iOff>=pCtx->nToken ) rc = SQLITE_DONE;
+  }
+
+  return rc;
+}
+
+/*
+** Set pIter->nScore to the score for the current entry.
+*/
+static void fts5SnippetCalculateScore(SnipIter *pIter){
+  int i;
+  int nScore = 0;
+  assert( pIter->iLast>=0 );
+
+  for(i=0; i<pIter->nPhrase; i++){
+    SnipPhrase *p = &pIter->aPhrase[i];
+    u64 mask = p->mask;
+    if( mask ){
+      u64 j;
+      nScore += 1000;
+      for(j=1; j & pIter->szmask; j<<=1){
+        if( mask & j ) nScore++;
+      }
+    }
+  }
+
+  pIter->nScore = nScore;
+}
+
+/*
+** Allocate a new snippet iter.
+*/
+static int fts5SnipIterNew(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int nToken,                     /* Number of tokens in snippets */
+  SnipIter **ppIter            /* OUT: New object */
+){
+  int i;                          /* Counter variable */
+  SnipIter *pIter;             /* New iterator object */
+  int nByte;                      /* Bytes of space to allocate */
+  int nPhrase;                    /* Number of phrases in query */
+
+  *ppIter = 0;
+  nPhrase = pApi->xPhraseCount(pFts);
+  nByte = sizeof(SnipIter) + nPhrase * sizeof(SnipPhrase);
+  pIter = (SnipIter*)sqlite3_malloc(nByte);
+  if( pIter==0 ) return SQLITE_NOMEM;
+  memset(pIter, 0, nByte);
+
+  pIter->nPhrase = nPhrase;
+  pIter->pApi = pApi;
+  pIter->pFts = pFts;
+  pIter->szmask = ((u64)1 << nToken) - 1;
+  assert( nToken<=63 );
+
+  for(i=0; i<nPhrase; i++){
+    pIter->aPhrase[i].nToken = pApi->xPhraseSize(pFts, i);
+  }
+
+  *ppIter = pIter;
+  return SQLITE_OK;
+}
+
+/*
+** Set the iterator to point to the first candidate snippet.
+*/
+static void fts5SnipIterFirst(SnipIter *pIter){
+  const Fts5ExtensionApi *pApi = pIter->pApi;
+  Fts5Context *pFts = pIter->pFts;
+  int i;                          /* Used to iterate through phrases */
+  SnipPhrase *pMin = 0;        /* Phrase with first match */
+
+  memset(pIter->aPhrase, 0, sizeof(SnipPhrase) * pIter->nPhrase);
+
+  for(i=0; i<pIter->nPhrase; i++){
+    SnipPhrase *p = &pIter->aPhrase[i];
+    p->nToken = pApi->xPhraseSize(pFts, i);
+    pApi->xPoslist(pFts, i, &p->i, &p->iPos);
+    if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ){
+      pMin = p;
+    }
+  }
+  assert( pMin );
+
+  pIter->iLast = pMin->iPos + pMin->nToken - 1;
+  pMin->mask = 0x01;
+  pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
+  fts5SnippetCalculateScore(pIter);
+}
+
+/*
+** Advance the snippet iterator to the next candidate snippet.
+*/
+static void fts5SnipIterNext(SnipIter *pIter){
+  const Fts5ExtensionApi *pApi = pIter->pApi;
+  Fts5Context *pFts = pIter->pFts;
+  int nPhrase = pIter->nPhrase;
+  int i;                          /* Used to iterate through phrases */
+  SnipPhrase *pMin = 0;
+
+  for(i=0; i<nPhrase; i++){
+    SnipPhrase *p = &pIter->aPhrase[i];
+    if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ) pMin = p;
+  }
+
+  if( pMin==0 ){
+    /* pMin==0 indicates that the SnipIter is at EOF. */
+    pIter->iLast = -1;
+  }else{
+    i64 nShift = pMin->iPos - pIter->iLast;
+    assert( nShift>=0 );
+    for(i=0; i<nPhrase; i++){
+      SnipPhrase *p = &pIter->aPhrase[i];
+      if( nShift>=63 ){
+        p->mask = 0;
+      }else{
+        p->mask = p->mask << (int)nShift;
+        p->mask &= pIter->szmask;
+      }
+    }
+
+    pIter->iLast = pMin->iPos;
+    pMin->mask |= 0x01;
+    fts5SnippetCalculateScore(pIter);
+    pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
+  }
+}
+
+static void fts5SnipIterFree(SnipIter *pIter){
+  if( pIter ){
+    sqlite3_free(pIter);
+  }
+}
+
+static int fts5SnippetText(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  SnipIter *pIter,             /* Snippet to write to buffer */
+  int nToken,                     /* Size of desired snippet in tokens */
+  const char *zStart,
+  const char *zFinal,
+  const char *zEllip,
+  Fts5Buffer *pBuf                /* Write output to this buffer */
+){
+  SnippetCtx ctx;
+  int i;
+  u64 all = 0;
+  const char *zCol;               /* Column text to extract snippet from */
+  int nCol;                       /* Size of column text in bytes */
+  int rc;
+  int nShift;
+
+  rc = pApi->xColumnText(pFts, FTS5_POS2COLUMN(pIter->iLast), &zCol, &nCol);
+  if( rc!=SQLITE_OK ) return rc;
+
+  /* At this point pIter->iLast is the offset of the last token in the
+  ** proposed snippet. However, in all cases pIter->iLast contains the
+  ** final token of one of the phrases. This makes the snippet look
+  ** unbalanced. For example:
+  **
+  **     "...x x x x x <b>term</b>..."
+  **
+  ** It is better to increase iLast a little so that the snippet looks
+  ** more like:
+  **
+  **     "...x x x <b>term</b> y y..."
+  **
+  ** The problem is that there is no easy way to discover whether or not
+  ** how many tokens are present in the column following "term". 
+  */
+
+  /* Set variable nShift to the number of tokens by which the snippet
+  ** should be shifted, assuming there are sufficient tokens to the right
+  ** of iLast in the column value.  */
+  for(i=0; i<pIter->nPhrase; i++){
+    int iToken;
+    for(iToken=0; iToken<pIter->aPhrase[i].nToken; iToken++){
+      all |= (pIter->aPhrase[i].mask << iToken);
+    }
+  }
+  for(i=nToken-1; i>=0; i--){
+    if( all & ((u64)1 << i) ) break;
+  }
+  assert( i>=0 );
+  nShift = (nToken - i) / 2;
+
+  memset(&ctx, 0, sizeof(SnippetCtx));
+  ctx.nToken = nToken + nShift;
+  ctx.iFirst = FTS5_POS2OFFSET(pIter->iLast) - nToken + 1;
+  if( ctx.iFirst<0 ){
+    nShift += ctx.iFirst;
+    if( nShift<0 ) nShift = 0;
+    ctx.iFirst = 0;
+  }
+  ctx.aiStart = (int*)sqlite3_malloc(sizeof(int) * ctx.nToken * 2);
+  if( ctx.aiStart==0 ) return SQLITE_NOMEM;
+  ctx.aiEnd = &ctx.aiStart[ctx.nToken];
+
+  rc = pApi->xTokenize(pFts, zCol, nCol, (void*)&ctx, fts5SnippetCallback);
+  if( rc==SQLITE_OK ){
+    int i1;                       /* First token from input to include */
+    int i2;                       /* Last token from input to include */
+
+    int iPrint;
+    int iMatchto;
+    int iLast;
+
+    int *aiStart = ctx.aiStart - ctx.iFirst;
+    int *aiEnd = ctx.aiEnd - ctx.iFirst;
+
+    /* Ideally we want to start the snippet with token (ctx.iFirst + nShift).
+    ** However, this is only possible if there are sufficient tokens within
+    ** the column. This block sets variables i1 and i2 to the first and last
+    ** input tokens to include in the snippet.  */
+    if( (ctx.iFirst + nShift + nToken)<=ctx.iSeen ){
+      i1 = ctx.iFirst + nShift;
+      i2 = i1 + nToken - 1;
+    }else{
+      i2 = ctx.iSeen;
+      i1 = ctx.iSeen - nToken + 1;
+      assert( i1>=0 || ctx.iFirst==0 );
+      if( i1<0 ) i1 = 0;
+    }
+
+    /* If required, append the preceding ellipsis. */
+    if( i1>0 ) sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%s", zEllip);
+
+    iLast = FTS5_POS2OFFSET(pIter->iLast);
+    iPrint = i1;
+    iMatchto = -1;
+
+    for(i=i1; i<=i2; i++){
+
+      /* Check if this is the first token of any phrase match. */
+      int ip;
+      for(ip=0; ip<pIter->nPhrase; ip++){
+        SnipPhrase *pPhrase = &pIter->aPhrase[ip];
+        u64 m = (1 << (iLast - i - pPhrase->nToken + 1));
+
+        if( i<=iLast && (pPhrase->mask & m) ){
+          if( iMatchto<0 ){
+            sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s%s",
+                aiStart[i] - aiStart[iPrint],
+                &zCol[aiStart[iPrint]],
+                zStart
+            );
+            iPrint = i;
+          }
+          if( i>iMatchto ) iMatchto = i + pPhrase->nToken - 1;
+        }
+      }
+
+      if( i==iMatchto ){
+        sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s%s",
+            aiEnd[i] - aiStart[iPrint],
+            &zCol[aiStart[iPrint]],
+            zFinal
+        );
+        iMatchto = -1;
+        iPrint = i+1;
+
+        if( i<i2 ){
+          sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s",
+              aiStart[i+1] - aiEnd[i],
+              &zCol[aiEnd[i]]
+          );
+        }
+      }
+    }
+
+    if( iPrint<=i2 ){
+      sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s", 
+          aiEnd[i2] - aiStart[iPrint], 
+          &zCol[aiStart[iPrint]]
+      );
+      if( iMatchto>=0 ){
+        sqlite3Fts5BufferAppendString(&rc, pBuf, zFinal);
+      }
+    }
+
+    /* If required, append the trailing ellipsis. */
+    if( i2<ctx.iSeen ) sqlite3Fts5BufferAppendString(&rc, pBuf, zEllip);
+  }
+
+  sqlite3_free(ctx.aiStart);
+  return rc;
+}
+
+/*
+** A default snippet() implementation. This is compatible with the FTS3
+** snippet() function.
+*/
+static void fts5SnippetFunction(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  sqlite3_context *pCtx,          /* Context for returning result/error */
+  int nVal,                       /* Number of values in apVal[] array */
+  sqlite3_value **apVal           /* Array of trailing arguments */
+){
+  const char *zStart = "<b>";
+  const char *zFinal = "</b>";
+  const char *zEllip = "<b>...</b>";
+  int nToken = -15;
+  int nAbs;
+  int rc;
+  SnipIter *pIter = 0;
+
+  if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
+  if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
+  if( nVal>=3 ) zEllip = (const char*)sqlite3_value_text(apVal[2]);
+  if( nVal>=4 ){
+    nToken = sqlite3_value_int(apVal[3]);
+    if( nToken==0 ) nToken = -15;
+  }
+  nAbs = nToken * (nToken<0 ? -1 : 1);
+
+  rc = fts5SnipIterNew(pApi, pFts, nAbs, &pIter);
+  if( rc==SQLITE_OK ){
+    Fts5Buffer buf;               /* Result buffer */
+    int nBestScore = 0;           /* Score of best snippet found */
+
+    for(fts5SnipIterFirst(pIter); 
+        pIter->iLast>=0; 
+        fts5SnipIterNext(pIter)
+    ){
+      if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
+    }
+    for(fts5SnipIterFirst(pIter); 
+        pIter->iLast>=0; 
+        fts5SnipIterNext(pIter)
+    ){
+      if( pIter->nScore==nBestScore ) break;
+    }
+
+    memset(&buf, 0, sizeof(Fts5Buffer));
+    rc = fts5SnippetText(pApi, pFts, pIter, nAbs, zStart, zFinal, zEllip, &buf);
+    if( rc==SQLITE_OK ){
+      sqlite3_result_text(pCtx, (const char*)buf.p, buf.n, SQLITE_TRANSIENT);
+    }
+    sqlite3_free(buf.p);
+  }
+
+  fts5SnipIterFree(pIter);
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+  }
+}
+
+
+/*
+** Context object passed by fts5GatherTotals() to xQueryPhrase callback
+** fts5GatherCallback().
+*/
+struct Fts5GatherCtx {
+  int nCol;                       /* Number of columns in FTS table */
+  int iPhrase;                    /* Phrase currently under investigation */
+  int *anVal;                     /* Array to populate */
+};
+
+/*
+** Callback used by fts5GatherTotals() with the xQueryPhrase() API.
+*/
+static int fts5GatherCallback(
+  const Fts5ExtensionApi *pApi, 
+  Fts5Context *pFts,
+  void *pUserData                 /* Pointer to Fts5GatherCtx object */
+){
+  struct Fts5GatherCtx *p = (struct Fts5GatherCtx*)pUserData;
+  int i = 0;
+  int iPrev = -1;
+  i64 iPos = 0;
+
+  while( 0==pApi->xPoslist(pFts, 0, &i, &iPos) ){
+    int iCol = FTS5_POS2COLUMN(iPos);
+    if( iCol!=iPrev ){
+      p->anVal[p->iPhrase * p->nCol + iCol]++;
+      iPrev = iCol;
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** This function returns a pointer to an array of integers containing entries
+** indicating the number of rows in the table for which each phrase features 
+** at least once in each column.
+**
+** If nCol is the number of matchable columns in the table, and nPhrase is
+** the number of phrases in the query, the array contains a total of
+** (nPhrase*nCol) entries.
+**
+** For phrase iPhrase and column iCol:
+**
+**   anVal[iPhrase * nCol + iCol]
+**
+** is set to the number of rows in the table for which column iCol contains 
+** at least one instance of phrase iPhrase.
+*/
+static int fts5GatherTotals(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int **panVal
+){
+  int rc = SQLITE_OK;
+  int *anVal = 0;
+  int i;                          /* For iterating through expression phrases */
+  int nPhrase = pApi->xPhraseCount(pFts);
+  int nCol = pApi->xColumnCount(pFts);
+  int nByte = nCol * nPhrase * sizeof(int);
+  struct Fts5GatherCtx sCtx;
+
+  sCtx.nCol = nCol;
+  anVal = sCtx.anVal = (int*)sqlite3_malloc(nByte);
+  if( anVal==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    memset(anVal, 0, nByte);
+  }
+
+  for(i=0; i<nPhrase && rc==SQLITE_OK; i++){
+    sCtx.iPhrase = i;
+    rc = pApi->xQueryPhrase(pFts, i, (void*)&sCtx, fts5GatherCallback);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(anVal);
+    anVal = 0;
+  }
+
+  *panVal = anVal;
+  return rc;
+}
+
+typedef struct Fts5Bm25Context Fts5Bm25Context;
+struct Fts5Bm25Context {
+  int nPhrase;                    /* Number of phrases in query */
+  int nCol;                       /* Number of columns in FTS table */
+  double *aIDF;                   /* Array of IDF values */
+  double *aAvg;                   /* Average size of each column in tokens */
+};
+
+static int fts5Bm25GetContext(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  Fts5Bm25Context **pp            /* OUT: Context object */
+){
+  Fts5Bm25Context *p;
+  int rc = SQLITE_OK;
+
+  p = pApi->xGetAuxdata(pFts, 0);
+  if( p==0 ){
+    int *anVal = 0;
+    int ic;                       /* For iterating through columns */
+    int ip;                       /* For iterating through phrases */
+    i64 nRow;                     /* Total number of rows in table */
+    int nPhrase = pApi->xPhraseCount(pFts);
+    int nCol = pApi->xColumnCount(pFts);
+    int nByte = sizeof(Fts5Bm25Context) 
+              + sizeof(double) * nPhrase * nCol       /* aIDF[] */
+              + sizeof(double) * nCol;                /* aAvg[] */
+
+    p = (Fts5Bm25Context*)sqlite3_malloc(nByte);
+    if( p==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(p, 0, nByte);
+      p->aAvg = (double*)&p[1];
+      p->aIDF = (double*)&p->aAvg[nCol];
+      p->nCol = nCol;
+      p->nPhrase = nPhrase;
+    }
+
+    if( rc==SQLITE_OK ){
+      rc = pApi->xRowCount(pFts, &nRow); 
+      assert( nRow>0 || rc!=SQLITE_OK );
+      if( nRow<2 ) nRow = 2;
+    }
+
+    for(ic=0; rc==SQLITE_OK && ic<nCol; ic++){
+      i64 nToken = 0;
+      rc = pApi->xColumnTotalSize(pFts, ic, &nToken);
+      p->aAvg[ic] = (double)nToken / (double)nRow;
+    }
+
+    if( rc==SQLITE_OK ){
+      rc = fts5GatherTotals(pApi, pFts, &anVal);
+    }
+    for(ic=0; ic<nCol; ic++){
+      for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){
+        /* Calculate the IDF (Inverse Document Frequency) for phrase ip
+        ** in column ic. This is done using the standard BM25 formula as
+        ** found on wikipedia:
+        **
+        **   IDF = log( (N - nHit + 0.5) / (nHit + 0.5) )
+        **
+        ** where "N" is the total number of documents in the set and nHit
+        ** is the number that contain at least one instance of the phrase
+        ** under consideration.
+        **
+        ** The problem with this is that if (N < 2*nHit), the IDF is 
+        ** negative. Which is undesirable. So the mimimum allowable IDF is
+        ** (1e-6) - roughly the same as a term that appears in just over
+        ** half of set of 5,000,000 documents.  */
+        int idx = ip * nCol + ic; /* Index in aIDF[] and anVal[] arrays */
+        int nHit = anVal[idx];    /* Number of docs matching "ic: ip" */
+
+        p->aIDF[idx] = log( (0.5 + nRow - nHit) / (0.5 + nHit) );
+        if( p->aIDF[idx]<=0.0 ) p->aIDF[idx] = 1e-6;
+        assert( p->aIDF[idx]>=0.0 );
+      }
+    }
+
+    sqlite3_free(anVal);
+    if( rc==SQLITE_OK ){
+      rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
+    }
+    if( rc!=SQLITE_OK ){
+      sqlite3_free(p);
+      p = 0;
+    }
+  }
+
+  *pp = p;
+  return rc;
+}
+
+static void fts5Bm25DebugContext(
+  int *pRc,                       /* IN/OUT: Return code */
+  Fts5Buffer *pBuf,               /* Buffer to populate */
+  Fts5Bm25Context *p              /* Context object to decode */
+){
+  int ip;
+  int ic;
+
+  sqlite3Fts5BufferAppendString(pRc, pBuf, "idf ");
+  if( p->nPhrase>1 || p->nCol>1 ){
+    sqlite3Fts5BufferAppendString(pRc, pBuf, "{");
+  }
+  for(ip=0; ip<p->nPhrase; ip++){
+    if( ip>0 ) sqlite3Fts5BufferAppendString(pRc, pBuf, " ");
+    if( p->nCol>1 ) sqlite3Fts5BufferAppendString(pRc, pBuf, "{");
+    for(ic=0; ic<p->nCol; ic++){
+      if( ic>0 ) sqlite3Fts5BufferAppendString(pRc, pBuf, " ");
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%f", p->aIDF[ip*p->nCol+ic]);
+    }
+    if( p->nCol>1 ) sqlite3Fts5BufferAppendString(pRc, pBuf, "}");
+  }
+  if( p->nPhrase>1 || p->nCol>1 ){
+    sqlite3Fts5BufferAppendString(pRc, pBuf, "}");
+  }
+
+  sqlite3Fts5BufferAppendString(pRc, pBuf, " avgdl ");
+  if( p->nCol>1 ) sqlite3Fts5BufferAppendString(pRc, pBuf, "{");
+  for(ic=0; ic<p->nCol; ic++){
+    if( ic>0 ) sqlite3Fts5BufferAppendString(pRc, pBuf, " ");
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%f", p->aAvg[ic]);
+  }
+  if( p->nCol>1 ) sqlite3Fts5BufferAppendString(pRc, pBuf, "}");
+}
+
+static void fts5Bm25DebugRow(
+  int *pRc, 
+  Fts5Buffer *pBuf, 
+  Fts5Bm25Context *p, 
+  const Fts5ExtensionApi *pApi, 
+  Fts5Context *pFts
+){
+}
+
+static void fts5Bm25Function(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  sqlite3_context *pCtx,          /* Context for returning result/error */
+  int nVal,                       /* Number of values in apVal[] array */
+  sqlite3_value **apVal           /* Array of trailing arguments */
+){
+  const double k1 = 1.2;
+  const double B = 0.75;
+  int rc = SQLITE_OK;
+  Fts5Bm25Context *p;
+
+  rc = fts5Bm25GetContext(pApi, pFts, &p);
+
+  if( rc==SQLITE_OK ){
+    /* If the bDebug flag is set, instead of returning a numeric rank, this
+    ** function returns a text value showing how the rank is calculated. */
+    Fts5Buffer debug;
+    int bDebug = (pApi->xUserData(pFts)!=0);
+    memset(&debug, 0, sizeof(Fts5Buffer));
+
+    int ip;
+    double score = 0.0;
+
+    if( bDebug ){
+      fts5Bm25DebugContext(&rc, &debug, p);
+      fts5Bm25DebugRow(&rc, &debug, p, pApi, pFts);
+    }
+
+    for(ip=0; rc==SQLITE_OK && ip<p->nPhrase; ip++){
+      int iPrev = 0;
+      int nHit = 0;
+      int i = 0;
+      i64 iPos = 0;
+
+      while( rc==SQLITE_OK ){
+        int bDone = pApi->xPoslist(pFts, ip, &i, &iPos);
+        int iCol = FTS5_POS2COLUMN(iPos);
+        if( (iCol!=iPrev || bDone) && nHit>0 ){
+          int sz = 0;
+          int idx = ip * p->nCol + iPrev;
+          double bm25;
+          rc = pApi->xColumnSize(pFts, iPrev, &sz);
+
+          bm25 = (p->aIDF[idx] * nHit * (k1+1.0)) /
+            (nHit + k1 * (1.0 - B + B * sz / p->aAvg[iPrev]));
+
+
+          score = score + bm25;
+          nHit = 0;
+        }
+        if( bDone ) break;
+        nHit++;
+        iPrev = iCol;
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      if( bDebug ){
+        sqlite3_result_text(pCtx, (const char*)debug.p, -1, SQLITE_TRANSIENT);
+      }else{
+        sqlite3_result_double(pCtx, score);
+      }
+    }
+    sqlite3_free(debug.p);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+  }
+}
+
+static int fts5TestCallback(
+  void *pContext,                 /* Pointer to Fts5Buffer object */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd,                       /* End offset of token */
+  int iPos                        /* Position offset of token */
+){
+  int rc = SQLITE_OK;
+  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
+  if( pBuf->n!=0 ){
+    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
+  }
+  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
+  return rc;
+}
+
+
+static void fts5TestFunction(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  sqlite3_context *pCtx,          /* Context for returning result/error */
+  int nVal,                       /* Number of values in apVal[] array */
+  sqlite3_value **apVal           /* Array of trailing arguments */
+){
+  Fts5Buffer s;                   /* Build up text to return here */
+  int nCol;                       /* Number of columns in table */
+  int nPhrase;                    /* Number of phrases in query */
+  i64 iRowid;                     /* Rowid of current row */
+  const char *zReq = 0;
+  int rc = SQLITE_OK;
+  int i;
+
+  if( nVal>=1 ){
+    zReq = (const char*)sqlite3_value_text(apVal[0]);
+  }
+
+  memset(&s, 0, sizeof(Fts5Buffer));
+  nCol = pApi->xColumnCount(pFts);
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columntotalsize ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntotalsize") ){
+    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
+    for(i=0; rc==SQLITE_OK && i<nCol; i++){
+      i64 colsz = 0;
+      rc = pApi->xColumnTotalSize(pFts, i, &colsz);
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
+    }
+    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " columncount ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " columnsize ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnsize") ){
+    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
+    for(i=0; rc==SQLITE_OK && i<nCol; i++){
+      int colsz = 0;
+      rc = pApi->xColumnSize(pFts, i, &colsz);
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
+    }
+    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " columntext ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntext") ){
+    for(i=0; rc==SQLITE_OK && i<nCol; i++){
+      const char *z;
+      int n;
+      rc = pApi->xColumnText(pFts, i, &z, &n);
+      if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " ");
+      sqlite3Fts5BufferAppendListElem(&rc, &s, z, n);
+    }
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasecount ");
+  }
+  nPhrase = pApi->xPhraseCount(pFts);
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasecount") ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nPhrase);
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasesize ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasesize") ){
+    if( nPhrase==1 ){
+      int nSize = pApi->xPhraseSize(pFts, 0);
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nSize);
+    }else{
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
+      for(i=0; i<nPhrase; i++){
+        int nSize = pApi->xPhraseSize(pFts, i);
+        sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", (i==0?"":" "), nSize);
+      }
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
+    }
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " poslist ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "poslist") ){
+    int bParen = 0;
+    Fts5Buffer s3;
+    memset(&s3, 0, sizeof(s3));
+
+
+    for(i=0; i<nPhrase; i++){
+      Fts5Buffer s2;                  /* List of positions for phrase/column */
+      int j = 0;
+      i64 iPos = 0;
+      int nElem = 0;
+
+      memset(&s2, 0, sizeof(s2));
+      while( 0==pApi->xPoslist(pFts, i, &j, &iPos) ){
+        int iOff = FTS5_POS2OFFSET(iPos);
+        int iCol = FTS5_POS2COLUMN(iPos);
+        if( nElem!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s2, " ");
+        sqlite3Fts5BufferAppendPrintf(&rc, &s2, "%d.%d", iCol, iOff);
+        nElem++;
+      }
+
+      if( i!=0 ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &s3, " ");
+      }
+      if( nElem==1 ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "%s", (const char*)s2.p);
+      }else{
+        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "{%s}", (const char*)s2.p);
+        bParen = 1;
+      }
+      sqlite3_free(s2.p);
+    }
+
+    if(zReq==0 && (nPhrase>1 || bParen) ){
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{%s}", (const char*)s3.p);
+    }else{
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p);
+    }
+    sqlite3_free(s3.p);
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " queryphrase ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "queryphrase") ){
+    int ic, ip;
+    int *anVal = 0;
+    Fts5Buffer buf1;
+    memset(&buf1, 0, sizeof(Fts5Buffer));
+
+    if( rc==SQLITE_OK ){
+      anVal = (int*)pApi->xGetAuxdata(pFts, 0);
+      if( anVal==0 ){
+        rc = fts5GatherTotals(pApi, pFts, &anVal);
+        if( rc==SQLITE_OK ){
+          rc = pApi->xSetAuxdata(pFts, (void*)anVal, sqlite3_free);
+        }
+      }
+    }
+
+    for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){
+      if( ip>0 ) sqlite3Fts5BufferAppendString(&rc, &buf1, " ");
+      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "{");
+      for(ic=0; ic<nCol; ic++){
+        int iVal = anVal[ip * nCol + ic];
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf1, "%s%d", ic==0?"":" ", iVal);
+      }
+      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "}");
+    }
+
+    if( zReq==0 ){
+      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf1.p, buf1.n);
+    }else{
+      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf1.p);
+    }
+    sqlite3_free(buf1.p);
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendString(&rc, &s, " rowid ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
+    iRowid = pApi->xRowid(pFts);
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendString(&rc, &s, " rowcount ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowcount") ){
+    i64 nRow;
+    rc = pApi->xRowCount(pFts, &nRow);
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", nRow);
+  }
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendString(&rc, &s, " tokenize ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
+    Fts5Buffer buf;
+    memset(&buf, 0, sizeof(buf));
+    for(i=0; rc==SQLITE_OK && i<nCol; i++){
+      const char *z;
+      int n;
+      rc = pApi->xColumnText(pFts, i, &z, &n);
+      if( rc==SQLITE_OK ){
+        Fts5Buffer buf1;
+        memset(&buf1, 0, sizeof(Fts5Buffer));
+        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
+        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
+        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
+        sqlite3_free(buf1.p);
+      }
+    }
+    if( zReq==0 ){
+      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
+    }else{
+      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
+    }
+    sqlite3_free(buf.p);
+  }
+
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+  sqlite3Fts5BufferFree(&s);
+}
+
+int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
+  struct Builtin {
+    const char *zFunc;            /* Function name (nul-terminated) */
+    void *pUserData;              /* User-data pointer */
+    fts5_extension_function xFunc;/* Callback function */
+    void (*xDestroy)(void*);      /* Destructor function */
+  } aBuiltin [] = {
+    { "bm25debug", (void*)1, fts5Bm25Function,    0 },
+    { "snippet",   0, fts5SnippetFunction, 0 },
+    { "fts5_test", 0, fts5TestFunction,    0 },
+    { "bm25",      0, fts5Bm25Function,    0 },
+  };
+
+  int rc = SQLITE_OK;             /* Return code */
+  int i;                          /* To iterate through builtin functions */
+
+  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
+    rc = sqlite3Fts5CreateAux(pGlobal, 
+        aBuiltin[i].zFunc,
+        aBuiltin[i].pUserData,
+        aBuiltin[i].xFunc,
+        aBuiltin[i].xDestroy
+    );
+  }
+
+  return rc;
+}
+
+

ext/fts5/fts5_buffer.c

@@ -0,0 +1,267 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+*/
+
+
+
+#include "fts5Int.h"
+
+int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
+  /* A no-op if an error has already occurred */
+  if( *pRc ) return 1;
+
+  if( (pBuf->n + nByte) > pBuf->nSpace ){
+    u8 *pNew;
+    int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
+    while( nNew<(pBuf->n + nByte) ){
+      nNew = nNew * 2;
+    }
+    pNew = sqlite3_realloc(pBuf->p, nNew);
+    if( pNew==0 ){
+      *pRc = SQLITE_NOMEM;
+      return 1;
+    }else{
+      pBuf->nSpace = nNew;
+      pBuf->p = pNew;
+    }
+  }
+  return 0;
+}
+
+/*
+** Encode value iVal as an SQLite varint and append it to the buffer object
+** pBuf. If an OOM error occurs, set the error code in p.
+*/
+void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
+  if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return;
+  pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal);
+}
+
+/*
+** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set 
+** the error code in p. If an error has already occurred when this function
+** is called, it is a no-op.
+*/
+void sqlite3Fts5BufferAppendBlob(
+  int *pRc,
+  Fts5Buffer *pBuf, 
+  int nData, 
+  const u8 *pData
+){
+  if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return;
+  memcpy(&pBuf->p[pBuf->n], pData, nData);
+  pBuf->n += nData;
+}
+
+/*
+** Append the nul-terminated string zStr to the buffer pBuf. This function
+** ensures that the byte following the buffer data is set to 0x00, even 
+** though this byte is not included in the pBuf->n count.
+*/
+void sqlite3Fts5BufferAppendString(
+  int *pRc,
+  Fts5Buffer *pBuf, 
+  const char *zStr
+){
+  int nStr = strlen(zStr);
+  if( sqlite3Fts5BufferGrow(pRc, pBuf, nStr+1) ) return;
+  sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr, (const u8*)zStr);
+  if( *pRc==SQLITE_OK ) pBuf->p[pBuf->n] = 0x00;
+}
+
+/*
+** Argument zFmt is a printf() style format string. This function performs
+** the printf() style processing, then appends the results to buffer pBuf.
+**
+** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte 
+** following the buffer data is set to 0x00, even though this byte is not
+** included in the pBuf->n count.
+*/ 
+void sqlite3Fts5BufferAppendPrintf(
+  int *pRc,
+  Fts5Buffer *pBuf, 
+  char *zFmt, ...
+){
+  if( *pRc==SQLITE_OK ){
+    char *zTmp;
+    va_list ap;
+    va_start(ap, zFmt);
+    zTmp = sqlite3_vmprintf(zFmt, ap);
+    va_end(ap);
+
+    if( zTmp==0 ){
+      *pRc = SQLITE_NOMEM;
+    }else{
+      sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
+      sqlite3_free(zTmp);
+    }
+  }
+}
+
+/*
+** Free any buffer allocated by pBuf. Zero the structure before returning.
+*/
+void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
+  sqlite3_free(pBuf->p);
+  memset(pBuf, 0, sizeof(Fts5Buffer));
+}
+
+/*
+** Zero the contents of the buffer object. But do not free the associated 
+** memory allocation.
+*/
+void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){
+  pBuf->n = 0;
+}
+
+/*
+** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
+** the error code in p. If an error has already occurred when this function
+** is called, it is a no-op.
+*/
+void sqlite3Fts5BufferSet(
+  int *pRc,
+  Fts5Buffer *pBuf, 
+  int nData, 
+  const u8 *pData
+){
+  pBuf->n = 0;
+  sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
+}
+
+int sqlite3Fts5PoslistNext64(
+  const u8 *a, int n,             /* Buffer containing poslist */
+  int *pi,                        /* IN/OUT: Offset within a[] */
+  i64 *piOff                      /* IN/OUT: Current offset */
+){
+  int i = *pi;
+  if( i>=n ){
+    /* EOF */
+    *piOff = -1;
+    return 1;  
+  }else{
+    i64 iOff = *piOff;
+    int iVal;
+    i += getVarint32(&a[i], iVal);
+    if( iVal==1 ){
+      i += getVarint32(&a[i], iVal);
+      iOff = ((i64)iVal) << 32;
+      i += getVarint32(&a[i], iVal);
+    }
+    *piOff = iOff + (iVal-2);
+    *pi = i;
+    return 0;
+  }
+}
+
+
+/*
+** Advance the iterator object passed as the only argument. Return true
+** if the iterator reaches EOF, or false otherwise.
+*/
+int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){
+  if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) 
+   || (pIter->iCol>=0 && (pIter->iPos >> 32) > pIter->iCol)
+  ){
+    pIter->bEof = 1;
+  }
+  return pIter->bEof;
+}
+
+int sqlite3Fts5PoslistReaderInit(
+  int iCol,                       /* If (iCol>=0), this column only */
+  const u8 *a, int n,             /* Poslist buffer to iterate through */
+  Fts5PoslistReader *pIter        /* Iterator object to initialize */
+){
+  memset(pIter, 0, sizeof(*pIter));
+  pIter->a = a;
+  pIter->n = n;
+  pIter->iCol = iCol;
+  do {
+    sqlite3Fts5PoslistReaderNext(pIter);
+  }while( pIter->bEof==0 && (pIter->iPos >> 32)<iCol );
+  return pIter->bEof;
+}
+
+int sqlite3Fts5PoslistWriterAppend(
+  Fts5Buffer *pBuf, 
+  Fts5PoslistWriter *pWriter,
+  i64 iPos
+){
+  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
+  int rc = SQLITE_OK;
+  if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
+    fts5BufferAppendVarint(&rc, pBuf, 1);
+    fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32));
+    pWriter->iPrev = (iPos & colmask);
+  }
+  fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2);
+  pWriter->iPrev = iPos;
+  return rc;
+}
+
+int sqlite3Fts5PoslistNext(
+  const u8 *a, int n,             /* Buffer containing poslist */
+  int *pi,                        /* IN/OUT: Offset within a[] */
+  int *piCol,                     /* IN/OUT: Current column */
+  int *piOff                      /* IN/OUT: Current token offset */
+){
+  int i = *pi;
+  int iVal;
+  if( i>=n ){
+    /* EOF */
+    return 1;  
+  }
+  i += getVarint32(&a[i], iVal);
+  if( iVal==1 ){
+    i += getVarint32(&a[i], iVal);
+    *piCol = iVal;
+    *piOff = 0;
+    i += getVarint32(&a[i], iVal);
+  }
+  *piOff += (iVal-2);
+  *pi = i;
+  return 0;
+}
+
+void sqlite3Fts5BufferAppendListElem(
+  int *pRc,                       /* IN/OUT: Error code */
+  Fts5Buffer *pBuf,               /* Buffer to append to */
+  const char *z, int n            /* Value to append to buffer */
+){
+  int bParen = (n==0);
+  int nMax = n*2 + 2 + 1;
+  u8 *pOut;
+  int i;
+
+  /* Ensure the buffer has space for the new list element */
+  if( sqlite3Fts5BufferGrow(pRc, pBuf, nMax) ) return;
+  pOut = &pBuf->p[pBuf->n];
+
+  /* Figure out if we need the enclosing {} */
+  for(i=0; i<n && bParen==0; i++){
+    if( z[i]=='"' || z[i]==' ' ){
+      bParen = 1;
+    }
+  }
+
+  if( bParen ) *pOut++ = '{';
+  for(i=0; i<n; i++){
+    *pOut++ = z[i];
+  }
+  if( bParen ) *pOut++ = '}';
+
+  pBuf->n = pOut - pBuf->p;
+  *pOut = '\0';
+}
+
+

ext/fts5/fts5_config.c

@@ -0,0 +1,328 @@
+/*
+** 2014 Jun 09
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is an SQLite module implementing full-text search.
+*/
+
+#include "fts5Int.h"
+
+/*
+** Convert an SQL-style quoted string into a normal string by removing
+** the quote characters.  The conversion is done in-place.  If the
+** input does not begin with a quote character, then this routine
+** is a no-op.
+**
+** Examples:
+**
+**     "abc"   becomes   abc
+**     'xyz'   becomes   xyz
+**     [pqr]   becomes   pqr
+**     `mno`   becomes   mno
+*/
+void sqlite3Fts5Dequote(char *z){
+  char quote;                     /* Quote character (if any ) */
+
+  quote = z[0];
+  if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
+    int iIn = 1;                  /* Index of next byte to read from input */
+    int iOut = 0;                 /* Index of next byte to write to output */
+
+    /* If the first byte was a '[', then the close-quote character is a ']' */
+    if( quote=='[' ) quote = ']';  
+
+    while( ALWAYS(z[iIn]) ){
+      if( z[iIn]==quote ){
+        if( z[iIn+1]!=quote ) break;
+        z[iOut++] = quote;
+        iIn += 2;
+      }else{
+        z[iOut++] = z[iIn++];
+      }
+    }
+    z[iOut] = '\0';
+  }
+}
+
+/*
+** Parse the "special" CREATE VIRTUAL TABLE directive and update
+** configuration object pConfig as appropriate.
+**
+** If successful, object pConfig is updated and SQLITE_OK returned. If
+** an error occurs, an SQLite error code is returned and an error message
+** may be left in *pzErr. It is the responsibility of the caller to
+** eventually free any such error message using sqlite3_free().
+*/
+static int fts5ConfigParseSpecial(
+  Fts5Config *pConfig,            /* Configuration object to update */
+  char *zCmd,                     /* Special command to parse */
+  char *zArg,                     /* Argument to parse */
+  char **pzErr                    /* OUT: Error message */
+){
+  if( sqlite3_stricmp(zCmd, "prefix")==0 ){
+    char *p;
+    if( pConfig->aPrefix ){
+      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
+      return SQLITE_ERROR;
+    }
+    pConfig->aPrefix = sqlite3_malloc(sizeof(int) * FTS5_MAX_PREFIX_INDEXES);
+    p = zArg;
+    while( p[0] ){
+      int nPre = 0;
+      while( p[0]==' ' ) p++;
+      while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
+        nPre = nPre*10 + (p[0] - '0');
+        p++;
+      }
+      while( p[0]==' ' ) p++;
+      if( p[0]==',' ){
+        p++;
+      }else if( p[0] ){
+        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
+        return SQLITE_ERROR;
+      }
+      if( nPre==0 || nPre>=1000 ){
+        *pzErr = sqlite3_mprintf("prefix length out of range: %d", nPre);
+        return SQLITE_ERROR;
+      }
+      pConfig->aPrefix[pConfig->nPrefix] = nPre;
+      pConfig->nPrefix++;
+    }
+    return SQLITE_OK;
+  }
+
+  *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
+  return SQLITE_ERROR;
+}
+
+/*
+** Duplicate the string passed as the only argument into a buffer allocated
+** by sqlite3_malloc().
+**
+** Return 0 if an OOM error is encountered.
+*/
+static char *fts5Strdup(const char *z){
+  return sqlite3_mprintf("%s", z);
+}
+
+void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);
+
+/*
+** Allocate an instance of the default tokenizer ("simple") at 
+** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
+** code if an error occurs.
+*/
+static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
+  const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
+  sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance */
+  int rc;                         /* Return code */
+
+  sqlite3Fts3SimpleTokenizerModule(&pMod);
+  rc = pMod->xCreate(0, 0, &pTokenizer);
+  if( rc==SQLITE_OK ){
+    pTokenizer->pModule = pMod;
+    pConfig->pTokenizer = pTokenizer;
+  }
+
+  return rc;
+}
+
+/*
+** Arguments nArg/azArg contain the string arguments passed to the xCreate
+** or xConnect method of the virtual table. This function attempts to 
+** allocate an instance of Fts5Config containing the results of parsing
+** those arguments.
+**
+** If successful, SQLITE_OK is returned and *ppOut is set to point to the
+** new Fts5Config object. If an error occurs, an SQLite error code is 
+** returned, *ppOut is set to NULL and an error message may be left in
+** *pzErr. It is the responsibility of the caller to eventually free any 
+** such error message using sqlite3_free().
+*/
+int sqlite3Fts5ConfigParse(
+  sqlite3 *db,
+  int nArg,                       /* Number of arguments */
+  const char **azArg,             /* Array of nArg CREATE VIRTUAL TABLE args */
+  Fts5Config **ppOut,             /* OUT: Results of parse */
+  char **pzErr                    /* OUT: Error message */
+){
+  int rc = SQLITE_OK;             /* Return code */
+  Fts5Config *pRet;               /* New object to return */
+
+  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
+  if( pRet==0 ) return SQLITE_NOMEM;
+  memset(pRet, 0, sizeof(Fts5Config));
+  pRet->db = db;
+
+  pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
+  pRet->zDb = fts5Strdup(azArg[1]);
+  pRet->zName = fts5Strdup(azArg[2]);
+  if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
+    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
+    rc = SQLITE_ERROR;
+  }else if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    int i;
+    for(i=3; rc==SQLITE_OK && i<nArg; i++){
+      char *zDup = fts5Strdup(azArg[i]);
+      if( zDup==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+
+        /* Check if this is a special directive - "cmd=arg" */
+        if( zDup[0]!='"' && zDup[0]!='\'' && zDup[0]!='[' && zDup[0]!='`' ){
+          char *p = zDup;
+          while( *p && *p!='=' ) p++;
+          if( *p ){
+            char *zArg = &p[1];
+            *p = '\0';
+            sqlite3Fts5Dequote(zArg);
+            rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
+            sqlite3_free(zDup);
+            zDup = 0;
+          }
+        }
+
+        /* If it is not a special directive, it must be a column name. In
+        ** this case, check that it is not the reserved column name "rank". */
+        if( zDup ){
+          sqlite3Fts5Dequote(zDup);
+          pRet->azCol[pRet->nCol++] = zDup;
+          if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
+            *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
+            rc = SQLITE_ERROR;
+          }
+        }
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
+    rc = fts5ConfigDefaultTokenizer(pRet);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3Fts5ConfigFree(pRet);
+    *ppOut = 0;
+  }
+  return rc;
+}
+
+/*
+** Free the configuration object passed as the only argument.
+*/
+void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
+  if( pConfig ){
+    int i;
+    if( pConfig->pTokenizer ){
+      pConfig->pTokenizer->pModule->xDestroy(pConfig->pTokenizer);
+    }
+    sqlite3_free(pConfig->zDb);
+    sqlite3_free(pConfig->zName);
+    for(i=0; i<pConfig->nCol; i++){
+      sqlite3_free(pConfig->azCol[i]);
+    }
+    sqlite3_free(pConfig->azCol);
+    sqlite3_free(pConfig->aPrefix);
+    sqlite3_free(pConfig);
+  }
+}
+
+/*
+** Call sqlite3_declare_vtab() based on the contents of the configuration
+** object passed as the only argument. Return SQLITE_OK if successful, or
+** an SQLite error code if an error occurs.
+*/
+int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
+  int i;
+  int rc;
+  char *zSql;
+  char *zOld;
+
+  zSql = (char*)sqlite3_mprintf("CREATE TABLE x(");
+  for(i=0; zSql && i<pConfig->nCol; i++){
+    zOld = zSql;
+    zSql = sqlite3_mprintf("%s%s%Q", zOld, (i==0?"":", "), pConfig->azCol[i]);
+    sqlite3_free(zOld);
+  }
+
+  if( zSql ){
+    zOld = zSql;
+    zSql = sqlite3_mprintf("%s, %Q HIDDEN, %s HIDDEN)", 
+        zOld, pConfig->zName, FTS5_RANK_NAME
+    );
+    sqlite3_free(zOld);
+  }
+
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_declare_vtab(pConfig->db, zSql);
+    sqlite3_free(zSql);
+  }
+  
+  return rc;
+}
+
+/*
+** Tokenize the text passed via the second and third arguments.
+**
+** The callback is invoked once for each token in the input text. The
+** arguments passed to it are, in order:
+**
+**     void *pCtx          // Copy of 4th argument to sqlite3Fts5Tokenize()
+**     const char *pToken  // Pointer to buffer containing token
+**     int nToken          // Size of token in bytes
+**     int iStart          // Byte offset of start of token within input text
+**     int iEnd            // Byte offset of end of token within input text
+**     int iPos            // Position of token in input (first token is 0)
+**
+** If the callback returns a non-zero value the tokenization is abandoned
+** and no further callbacks are issued. 
+**
+** This function returns SQLITE_OK if successful or an SQLite error code
+** if an error occurs. If the tokenization was abandoned early because
+** the callback returned SQLITE_DONE, this is not an error and this function
+** still returns SQLITE_OK. Or, if the tokenization was abandoned early
+** because the callback returned another non-zero value, it is assumed
+** to be an SQLite error code and returned to the caller.
+*/
+int sqlite3Fts5Tokenize(
+  Fts5Config *pConfig,            /* FTS5 Configuration object */
+  const char *pText, int nText,   /* Text to tokenize */
+  void *pCtx,                     /* Context passed to xToken() */
+  int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
+){
+  const sqlite3_tokenizer_module *pMod = pConfig->pTokenizer->pModule;
+  sqlite3_tokenizer_cursor *pCsr = 0;
+  int rc;
+
+  rc = pMod->xOpen(pConfig->pTokenizer, pText, nText, &pCsr);
+  assert( rc==SQLITE_OK || pCsr==0 );
+  if( rc==SQLITE_OK ){
+    const char *pToken;           /* Pointer to token buffer */
+    int nToken;                   /* Size of token in bytes */
+    int iStart, iEnd, iPos;       /* Start, end and position of token */
+    pCsr->pTokenizer = pConfig->pTokenizer;
+    for(rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos);
+        rc==SQLITE_OK;
+        rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos)
+    ){
+      if( (rc = xToken(pCtx, pToken, nToken, iStart, iEnd, iPos)) ) break;
+    }
+    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+    pMod->xClose(pCsr);
+  }
+  return rc;
+}
+
+

ext/fts5/fts5_storage.c

@@ -0,0 +1,740 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+
+#include "fts5Int.h"
+
+struct Fts5Storage {
+  Fts5Config *pConfig;
+  Fts5Index *pIndex;
+  i64 nTotalRow;                  /* Total number of rows in FTS table */
+  i64 *aTotalSize;                /* Total sizes of each column */ 
+  sqlite3_stmt *aStmt[11];
+};
+
+
+#if FTS5_STMT_SCAN_ASC!=0 
+# error "FTS5_STMT_SCAN_ASC mismatch" 
+#endif
+#if FTS5_STMT_SCAN_DESC!=1 
+# error "FTS5_STMT_SCAN_DESC mismatch" 
+#endif
+#if FTS5_STMT_LOOKUP!=2
+# error "FTS5_STMT_LOOKUP mismatch" 
+#endif
+#if FTS5_STMT_SORTER_DESC!=3
+# error "FTS5_STMT_SORTER_DESC mismatch" 
+#endif
+#if FTS5_STMT_SORTER_ASC!=4
+# error "FTS5_STMT_SORTER_ASC mismatch" 
+#endif
+
+#define FTS5_STMT_INSERT_CONTENT  5
+#define FTS5_STMT_REPLACE_CONTENT 6
+
+#define FTS5_STMT_DELETE_CONTENT  7
+#define FTS5_STMT_REPLACE_DOCSIZE 8
+#define FTS5_STMT_DELETE_DOCSIZE  9
+#define FTS5_STMT_LOOKUP_DOCSIZE  10
+
+/*
+** Prepare the two insert statements - Fts5Storage.pInsertContent and
+** Fts5Storage.pInsertDocsize - if they have not already been prepared.
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs.
+*/
+static int fts5StorageGetStmt(
+  Fts5Storage *p,                 /* Storage handle */
+  int eStmt,                      /* FTS5_STMT_XXX constant */
+  sqlite3_stmt **ppStmt           /* OUT: Prepared statement handle */
+){
+  int rc = SQLITE_OK;
+
+  assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
+  if( p->aStmt[eStmt]==0 ){
+    const char *azStmt[] = {
+      "SELECT * FROM %Q.'%q_content' ORDER BY id ASC",  /* SCAN_ASC */
+      "SELECT * FROM %Q.'%q_content' ORDER BY id DESC", /* SCAN_DESC */
+      "SELECT * FROM %Q.'%q_content' WHERE rowid=?",    /* LOOKUP  */
+
+      /* SORTER_DESC and SORTER_ASC: */
+      "SELECT rowid, \"%s\" FROM %Q.%Q ORDER BY +" FTS5_RANK_NAME " DESC",
+      "SELECT rowid, \"%s\" FROM %Q.%Q ORDER BY +" FTS5_RANK_NAME " ASC",
+
+      "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
+      "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
+      "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
+      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
+      "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
+
+      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
+    };
+    Fts5Config *pConfig = p->pConfig;
+    char *zSql = 0;
+
+    if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){
+      int nCol = pConfig->nCol + 1;
+      char *zBind;
+      int i;
+
+      zBind = sqlite3_malloc(1 + nCol*2);
+      if( zBind ){
+        for(i=0; i<nCol; i++){
+          zBind[i*2] = '?';
+          zBind[i*2 + 1] = ',';
+        }
+        zBind[i*2-1] = '\0';
+        zSql = sqlite3_mprintf(azStmt[eStmt],pConfig->zDb,pConfig->zName,zBind);
+        sqlite3_free(zBind);
+      }
+    }else if( eStmt==FTS5_STMT_SORTER_ASC || eStmt==FTS5_STMT_SORTER_DESC ){
+      zSql = sqlite3_mprintf(azStmt[eStmt], 
+          pConfig->zName, pConfig->zDb, pConfig->zName
+      );
+    }else{
+      zSql = sqlite3_mprintf(azStmt[eStmt], pConfig->zDb, pConfig->zName);
+    }
+
+    if( zSql==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->aStmt[eStmt], 0);
+      sqlite3_free(zSql);
+    }
+  }
+
+  *ppStmt = p->aStmt[eStmt];
+  return rc;
+}
+
+/*
+** Drop the shadow table with the postfix zPost (e.g. "content"). Return
+** SQLITE_OK if successful or an SQLite error code otherwise.
+*/
+int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){
+  int rc;
+  char *zSql = sqlite3_mprintf("DROP TABLE IF EXISTS %Q.'%q_%q'",
+      pConfig->zDb, pConfig->zName, zPost
+  );
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_exec(pConfig->db, zSql, 0, 0, 0);
+    sqlite3_free(zSql);
+  }
+  return rc;
+}
+
+/*
+** Create the shadow table named zPost, with definition zDefn. Return
+** SQLITE_OK if successful, or an SQLite error code otherwise.
+*/
+int sqlite3Fts5CreateTable(
+  Fts5Config *pConfig,            /* FTS5 configuration */
+  const char *zPost,              /* Shadow table to create (e.g. "content") */
+  const char *zDefn,              /* Columns etc. for shadow table */
+  char **pzErr                    /* OUT: Error message */
+){
+  int rc;
+  char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)",
+      pConfig->zDb, pConfig->zName, zPost, zDefn
+  );
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    char *zErr = 0;
+    assert( *pzErr==0 );
+    rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
+    if( zErr ){
+      *pzErr = sqlite3_mprintf(
+          "fts5: error creating shadow table %q_%s: %s", 
+          pConfig->zName, zPost, zErr
+      );
+      sqlite3_free(zErr);
+    }
+    sqlite3_free(zSql);
+  }
+  return rc;
+}
+
+/*
+** Open a new Fts5Index handle. If the bCreate argument is true, create
+** and initialize the underlying tables 
+**
+** If successful, set *pp to point to the new object and return SQLITE_OK.
+** Otherwise, set *pp to NULL and return an SQLite error code.
+*/
+int sqlite3Fts5StorageOpen(
+  Fts5Config *pConfig, 
+  Fts5Index *pIndex, 
+  int bCreate, 
+  Fts5Storage **pp,
+  char **pzErr                    /* OUT: Error message */
+){
+  int rc;
+  Fts5Storage *p;                 /* New object */
+  int nByte;                      /* Bytes of space to allocate */
+
+  nByte = sizeof(Fts5Storage)               /* Fts5Storage object */
+        + pConfig->nCol * sizeof(i64);      /* Fts5Storage.aTotalSize[] */
+  *pp = p = (Fts5Storage*)sqlite3_malloc(nByte);
+  if( !p ) return SQLITE_NOMEM;
+
+  memset(p, 0, nByte);
+  p->aTotalSize = (i64*)&p[1];
+  p->pConfig = pConfig;
+  p->pIndex = pIndex;
+
+  if( bCreate ){
+    int i;
+    char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
+    if( zDefn==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
+      for(i=0; i<pConfig->nCol; i++){
+        iOff += sprintf(&zDefn[iOff], ", c%d", i);
+      }
+      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, pzErr);
+    }
+    sqlite3_free(zDefn);
+    if( rc==SQLITE_OK ){
+      rc = sqlite3Fts5CreateTable(
+          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", pzErr
+      );
+    }
+  }
+
+  if( rc ){
+    sqlite3Fts5StorageClose(p, 0);
+    *pp = 0;
+  }
+  return rc;
+}
+
+/*
+** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
+*/
+int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy){
+  int rc = SQLITE_OK;
+  int i;
+
+  /* Finalize all SQL statements */
+  for(i=0; i<ArraySize(p->aStmt); i++){
+    sqlite3_finalize(p->aStmt[i]);
+  }
+
+  /* If required, remove the shadow tables from the database */
+  if( bDestroy ){
+    rc = sqlite3Fts5DropTable(p->pConfig, "content");
+    if( rc==SQLITE_OK ) sqlite3Fts5DropTable(p->pConfig, "docsize");
+  }
+
+  sqlite3_free(p);
+  return rc;
+}
+
+typedef struct Fts5InsertCtx Fts5InsertCtx;
+struct Fts5InsertCtx {
+  Fts5Storage *pStorage;
+  int iCol;
+  int szCol;                      /* Size of column value in tokens */
+};
+
+/*
+** Tokenization callback used when inserting tokens into the FTS index.
+*/
+static int fts5StorageInsertCallback(
+  void *pContext,                 /* Pointer to Fts5InsertCtx object */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd,                       /* End offset of token */
+  int iPos                        /* Position offset of token */
+){
+  Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
+  Fts5Index *pIdx = pCtx->pStorage->pIndex;
+  pCtx->szCol = iPos+1;
+  sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
+  return SQLITE_OK;
+}
+
+/*
+** If a row with rowid iDel is present in the %_content table, add the
+** delete-markers to the FTS index necessary to delete it. Do not actually
+** remove the %_content row at this time though.
+*/
+static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
+  Fts5Config *pConfig = p->pConfig;
+  sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
+  int rc;                         /* Return code */
+
+  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek);
+  if( rc==SQLITE_OK ){
+    int rc2;
+    sqlite3_bind_int64(pSeek, 1, iDel);
+    if( sqlite3_step(pSeek)==SQLITE_ROW ){
+      int iCol;
+      Fts5InsertCtx ctx;
+      ctx.pStorage = p;
+      ctx.iCol = -1;
+      sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
+      for(iCol=1; iCol<=pConfig->nCol; iCol++){
+        rc = sqlite3Fts5Tokenize(pConfig, 
+            (const char*)sqlite3_column_text(pSeek, iCol),
+            sqlite3_column_bytes(pSeek, iCol),
+            (void*)&ctx,
+            fts5StorageInsertCallback
+        );
+        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+      }
+      p->nTotalRow--;
+    }
+    rc2 = sqlite3_reset(pSeek);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  return rc;
+}
+
+
+/*
+** Insert a record into the %_docsize table. Specifically, do:
+**
+**   INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
+*/
+static int fts5StorageInsertDocsize(
+  Fts5Storage *p,                 /* Storage module to write to */
+  i64 iRowid,                     /* id value */
+  Fts5Buffer *pBuf                /* sz value */
+){
+  sqlite3_stmt *pReplace = 0;
+  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pReplace, 1, iRowid);
+    sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+    sqlite3_step(pReplace);
+    rc = sqlite3_reset(pReplace);
+  }
+  return rc;
+}
+
+/*
+** Load the contents of the "averages" record from disk into the
+** p->nTotalRow and p->aTotalSize[] variables.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs.
+*/
+static int fts5StorageLoadTotals(Fts5Storage *p){
+  int nCol = p->pConfig->nCol;
+  Fts5Buffer buf;
+  int rc;
+  memset(&buf, 0, sizeof(buf));
+
+  memset(p->aTotalSize, 0, sizeof(i64) * nCol);
+  p->nTotalRow = 0;
+  rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf);
+  if( rc==SQLITE_OK && buf.n ){
+    int i = 0;
+    int iCol;
+    i += getVarint(&buf.p[i], (u64*)&p->nTotalRow);
+    for(iCol=0; i<buf.n && iCol<nCol; iCol++){
+      i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
+    }
+  }
+  sqlite3_free(buf.p);
+
+  return rc;
+}
+
+/*
+** Store the current contents of the p->nTotalRow and p->aTotalSize[] 
+** variables in the "averages" record on disk.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs.
+*/
+static int fts5StorageSaveTotals(Fts5Storage *p){
+  int nCol = p->pConfig->nCol;
+  int i;
+  Fts5Buffer buf;
+  int rc = SQLITE_OK;
+  memset(&buf, 0, sizeof(buf));
+
+  sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
+  for(i=0; i<nCol; i++){
+    sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
+  }
+  sqlite3_free(buf.p);
+
+  return rc;
+}
+
+/*
+** Remove a row from the FTS table.
+*/
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
+  int rc;
+  sqlite3_stmt *pDel;
+
+  rc = fts5StorageLoadTotals(p);
+
+  /* Delete the index records */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageDeleteFromIndex(p, iDel);
+  }
+
+  /* Delete the %_docsize record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel);
+  }
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pDel, 1, iDel);
+    sqlite3_step(pDel);
+    rc = sqlite3_reset(pDel);
+  }
+
+  /* Delete the %_content record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel);
+  }
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pDel, 1, iDel);
+    sqlite3_step(pDel);
+    rc = sqlite3_reset(pDel);
+  }
+
+  /* Write the averages record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageSaveTotals(p);
+  }
+
+  return rc;
+}
+
+/*
+** Insert a new row into the FTS table.
+*/
+int sqlite3Fts5StorageInsert(
+  Fts5Storage *p,                 /* Storage module to write to */
+  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
+  int eConflict,                  /* on conflict clause */
+  i64 *piRowid                    /* OUT: rowid of new record */
+){
+  Fts5Config *pConfig = p->pConfig;
+  int rc = SQLITE_OK;             /* Return code */
+  sqlite3_stmt *pInsert;          /* Statement used to write %_content table */
+  int eStmt;                      /* Type of statement used on %_content */
+  int i;                          /* Counter variable */
+  Fts5InsertCtx ctx;              /* Tokenization callback context object */
+  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
+
+  memset(&buf, 0, sizeof(Fts5Buffer));
+  rc = fts5StorageLoadTotals(p);
+
+  /* Insert the new row into the %_content table. */
+  if( rc==SQLITE_OK ){
+    if( eConflict==SQLITE_REPLACE ){
+      eStmt = FTS5_STMT_REPLACE_CONTENT;
+      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
+        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
+      }
+    }else{
+      eStmt = FTS5_STMT_INSERT_CONTENT;
+    }
+  }
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageGetStmt(p, eStmt, &pInsert);
+  }
+  for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
+    rc = sqlite3_bind_value(pInsert, i, apVal[i]);
+  }
+  if( rc==SQLITE_OK ){
+    sqlite3_step(pInsert);
+    rc = sqlite3_reset(pInsert);
+  }
+  *piRowid = sqlite3_last_insert_rowid(pConfig->db);
+
+  /* Add new entries to the FTS index */
+  sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
+  ctx.pStorage = p;
+  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
+    ctx.szCol = 0;
+    rc = sqlite3Fts5Tokenize(pConfig, 
+        (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
+        sqlite3_value_bytes(apVal[ctx.iCol+2]),
+        (void*)&ctx,
+        fts5StorageInsertCallback
+    );
+    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
+    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
+  }
+  p->nTotalRow++;
+
+  /* Write the %_docsize record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageInsertDocsize(p, *piRowid, &buf);
+  }
+  sqlite3_free(buf.p);
+
+  /* Write the averages record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageSaveTotals(p);
+  }
+
+  return rc;
+}
+
+static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
+  Fts5Config *pConfig = p->pConfig;
+  char *zSql;
+  int rc;
+
+  zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", 
+      pConfig->zDb, pConfig->zName, zSuffix
+  );
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    sqlite3_stmt *pCnt = 0;
+    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
+    if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCnt) ){
+      *pnRow = sqlite3_column_int64(pCnt, 0);
+    }
+    rc = sqlite3_finalize(pCnt);
+  }
+
+  sqlite3_free(zSql);
+  return rc;
+}
+
+/*
+** Context object used by sqlite3Fts5StorageIntegrity().
+*/
+typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
+struct Fts5IntegrityCtx {
+  i64 iRowid;
+  int iCol;
+  int szCol;
+  u64 cksum;
+  Fts5Config *pConfig;
+};
+
+/*
+** Tokenization callback used by integrity check.
+*/
+static int fts5StorageIntegrityCallback(
+  void *pContext,                 /* Pointer to Fts5InsertCtx object */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd,                       /* End offset of token */
+  int iPos                        /* Position offset of token */
+){
+  Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
+  pCtx->cksum ^= sqlite3Fts5IndexCksum(
+      pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken
+  );
+  pCtx->szCol = iPos+1;
+  return SQLITE_OK;
+}
+
+/*
+** Check that the contents of the FTS index match that of the %_content
+** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
+** some other SQLite error code if an error occurs while attempting to
+** determine this.
+*/
+int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
+  Fts5Config *pConfig = p->pConfig;
+  int rc;                         /* Return code */
+  int *aColSize;                  /* Array of size pConfig->nCol */
+  i64 *aTotalSize;                /* Array of size pConfig->nCol */
+  Fts5IntegrityCtx ctx;
+  sqlite3_stmt *pScan;
+
+  memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
+  ctx.pConfig = p->pConfig;
+  aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
+  if( !aTotalSize ) return SQLITE_NOMEM;
+  aColSize = (int*)&aTotalSize[pConfig->nCol];
+  memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
+
+  /* Generate the expected index checksum based on the contents of the
+  ** %_content table. This block stores the checksum in ctx.cksum. */
+  rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan);
+  if( rc==SQLITE_OK ){
+    int rc2;
+    while( SQLITE_ROW==sqlite3_step(pScan) ){
+      int i;
+      ctx.iRowid = sqlite3_column_int64(pScan, 0);
+      ctx.szCol = 0;
+      rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
+      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
+        ctx.iCol = i;
+        rc = sqlite3Fts5Tokenize(
+            pConfig, 
+            (const char*)sqlite3_column_text(pScan, i+1),
+            sqlite3_column_bytes(pScan, i+1),
+            (void*)&ctx,
+            fts5StorageIntegrityCallback
+        );
+        if( ctx.szCol!=aColSize[i] ) rc = SQLITE_CORRUPT_VTAB;
+        aTotalSize[i] += ctx.szCol;
+      }
+      if( rc!=SQLITE_OK ) break;
+    }
+    rc2 = sqlite3_reset(pScan);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  /* Test that the "totals" (sometimes called "averages") record looks Ok */
+  if( rc==SQLITE_OK ){
+    int i;
+    rc = fts5StorageLoadTotals(p);
+    for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
+      if( p->aTotalSize[i]!=aTotalSize[i] ) rc = SQLITE_CORRUPT_VTAB;
+    }
+  }
+
+  /* Check that the %_docsize and %_content tables contain the expected
+  ** number of rows.  */
+  if( rc==SQLITE_OK ){
+    i64 nRow;
+    rc = fts5StorageCount(p, "content", &nRow);
+    if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;
+  }
+  if( rc==SQLITE_OK ){
+    i64 nRow;
+    rc = fts5StorageCount(p, "docsize", &nRow);
+    if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;
+  }
+
+  /* Pass the expected checksum down to the FTS index module. It will
+  ** verify, amongst other things, that it matches the checksum generated by
+  ** inspecting the index itself.  */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum);
+  }
+
+  sqlite3_free(aTotalSize);
+  return rc;
+}
+
+/*
+** Obtain an SQLite statement handle that may be used to read data from the
+** %_content table.
+*/
+int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **pp){
+  int rc;
+  assert( eStmt==FTS5_STMT_SCAN_ASC 
+       || eStmt==FTS5_STMT_SCAN_DESC
+       || eStmt==FTS5_STMT_LOOKUP
+       || eStmt==FTS5_STMT_SORTER_DESC
+       || eStmt==FTS5_STMT_SORTER_ASC
+  );
+  rc = fts5StorageGetStmt(p, eStmt, pp);
+  if( rc==SQLITE_OK ){
+    assert( p->aStmt[eStmt]==*pp );
+    p->aStmt[eStmt] = 0;
+  }
+  return rc;
+}
+
+/*
+** Release an SQLite statement handle obtained via an earlier call to
+** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
+** must match that passed to the sqlite3Fts5StorageStmt() call.
+*/
+void sqlite3Fts5StorageStmtRelease(
+  Fts5Storage *p, 
+  int eStmt, 
+  sqlite3_stmt *pStmt
+){
+  assert( eStmt==FTS5_STMT_SCAN_ASC
+       || eStmt==FTS5_STMT_SCAN_DESC
+       || eStmt==FTS5_STMT_LOOKUP
+       || eStmt==FTS5_STMT_SORTER_DESC
+       || eStmt==FTS5_STMT_SORTER_ASC
+  );
+  if( p->aStmt[eStmt]==0 ){
+    sqlite3_reset(pStmt);
+    p->aStmt[eStmt] = pStmt;
+  }else{
+    sqlite3_finalize(pStmt);
+  }
+}
+
+static int fts5StorageDecodeSizeArray(
+  int *aCol, int nCol,            /* Array to populate */
+  const u8 *aBlob, int nBlob      /* Record to read varints from */
+){
+  int i;
+  int iOff = 0;
+  for(i=0; i<nCol; i++){
+    if( iOff>=nBlob ) return 1;
+    iOff += getVarint32(&aBlob[iOff], aCol[i]);
+  }
+  return (iOff!=nBlob);
+}
+
+/*
+** Argument aCol points to an array of integers containing one entry for
+** each table column. This function reads the %_docsize record for the
+** specified rowid and populates aCol[] with the results.
+**
+** An SQLite error code is returned if an error occurs, or SQLITE_OK
+** otherwise.
+*/
+int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
+  int nCol = p->pConfig->nCol;
+  sqlite3_stmt *pLookup = 0;
+  int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup);
+  if( rc==SQLITE_OK ){
+    int bCorrupt = 1;
+    sqlite3_bind_int64(pLookup, 1, iRowid);
+    if( SQLITE_ROW==sqlite3_step(pLookup) ){
+      const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
+      int nBlob = sqlite3_column_bytes(pLookup, 0);
+      if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
+        bCorrupt = 0;
+      }
+    }
+    rc = sqlite3_reset(pLookup);
+    if( bCorrupt && rc==SQLITE_OK ){
+      rc = SQLITE_CORRUPT_VTAB;
+    }
+  }
+  return rc;
+}
+
+int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
+  int rc = fts5StorageLoadTotals(p);
+  if( rc==SQLITE_OK ){
+    *pnToken = p->aTotalSize[iCol];
+  }
+  return rc;
+}
+
+int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
+  int rc = fts5StorageLoadTotals(p);
+  if( rc==SQLITE_OK ){
+    *pnRow = p->nTotalRow;
+  }
+  return rc;
+}
+

ext/fts5/fts5parse.y

@@ -0,0 +1,155 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+
+
+// All token codes are small integers with #defines that begin with "TK_"
+%token_prefix FTS5_
+
+// The type of the data attached to each token is Token.  This is also the
+// default type for non-terminals.
+//
+%token_type {Fts5Token}
+%default_type {Fts5Token}
+
+// The generated parser function takes a 4th argument as follows:
+%extra_argument {Fts5Parse *pParse}
+
+// This code runs whenever there is a syntax error
+//
+%syntax_error {
+  sqlite3Fts5ParseError(
+    pParse, "fts5: syntax error near \"%.*s\"",TOKEN.n,TOKEN.p
+  );
+}
+%stack_overflow {
+  assert( 0 );
+}
+
+// The name of the generated procedure that implements the parser
+// is as follows:
+%name sqlite3Fts5Parser
+
+// The following text is included near the beginning of the C source
+// code file that implements the parser.
+//
+%include {
+#include "fts5Int.h"
+#include "fts5parse.h"
+
+/*
+** Disable all error recovery processing in the parser push-down
+** automaton.
+*/
+#define YYNOERRORRECOVERY 1
+
+/*
+** Make yytestcase() the same as testcase()
+*/
+#define yytestcase(X) testcase(X)
+
+} // end %include
+
+%left OR.
+%left AND.
+%left NOT.
+%left COLON.
+
+input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
+
+%type cnearset    {Fts5ExprNode*}
+%type expr        {Fts5ExprNode*}
+%type exprlist    {Fts5ExprNode*}
+%destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
+%destructor expr     { sqlite3Fts5ParseNodeFree($$); }
+%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
+
+expr(A) ::= expr(X) AND expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
+}
+expr(A) ::= expr(X) OR expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
+}
+expr(A) ::= expr(X) NOT expr(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
+}
+
+expr(A) ::= LP expr(X) RP. {A = X;}
+expr(A) ::= exprlist(X).   {A = X;}
+
+exprlist(A) ::= cnearset(X). {A = X;}
+exprlist(A) ::= exprlist(X) cnearset(Y). {
+  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
+}
+
+cnearset(A) ::= nearset(X). { 
+  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
+}
+cnearset(A) ::= STRING(X) COLON nearset(Y). { 
+  sqlite3Fts5ParseSetColumn(pParse, Y, &X);
+  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
+}
+
+%type nearset     {Fts5ExprNearset*}
+%type nearphrases {Fts5ExprNearset*}
+%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
+%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
+
+nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
+nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
+  sqlite3Fts5ParseNear(pParse, &X);
+  sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
+  A = Y;
+}
+
+nearphrases(A) ::= phrase(X). { 
+  A = sqlite3Fts5ParseNearset(pParse, 0, X); 
+}
+nearphrases(A) ::= nearphrases(X) phrase(Y). {
+  A = sqlite3Fts5ParseNearset(pParse, X, Y);
+}
+
+/*
+** The optional ", <integer>" at the end of the NEAR() arguments.
+*/
+neardist_opt(A) ::= . { A.p = 0; A.n = 0; }
+neardist_opt(A) ::= COMMA STRING(X). { A = X; }
+
+/*
+** A phrase. A set of primitives connected by "+" operators. Examples:
+**
+**     "the" + "quick brown" + fo *
+**     "the quick brown fo" *
+**     the+quick+brown+fo*
+*/
+%type phrase {Fts5ExprPhrase*}
+%destructor phrase { sqlite3Fts5ParsePhraseFree($$); }
+
+phrase(A) ::= phrase(X) PLUS STRING(Y) star_opt(Z). { 
+  A = sqlite3Fts5ParseTerm(pParse, X, &Y, Z);
+}
+phrase(A) ::= STRING(Y) star_opt(Z). { 
+  A = sqlite3Fts5ParseTerm(pParse, 0, &Y, Z);
+}
+
+/*
+** Optional "*" character.
+*/
+%type star_opt {int}
+
+star_opt(A) ::= STAR. { A = 1; }
+star_opt(A) ::= . { A = 0; }
+
+
+
+

main.mk

@@ -47,6 +47,7 @@
 TCCX =  $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) 
 TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
 TCCX += -I$(TOP)/ext/async
+TCCX += -I$(TOP)/ext/fts5
 
 # Object files for the SQLite library.
 #
@@ -71,6 +72,15 @@ LIBOBJ+= vdbe.o parse.o \
          vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
 	 vdbetrace.o wal.o walker.o where.o utf.o vtab.o
 
+LIBOBJ += fts5.o
+LIBOBJ += fts5_aux.o
+LIBOBJ += fts5_buffer.o
+LIBOBJ += fts5_config.o
+LIBOBJ += fts5_expr.o
+LIBOBJ += fts5_index.o
+LIBOBJ += fts5_storage.o
+LIBOBJ += fts5parse.o
+
 
 
 # All of the source code files.
@@ -214,6 +224,18 @@ SRC += \
   $(TOP)/ext/rtree/rtree.h \
   $(TOP)/ext/rtree/rtree.c
 
+SRC += \
+   $(TOP)/ext/fts5/fts5.h \
+   $(TOP)/ext/fts5/fts5Int.h \
+   $(TOP)/ext/fts5/fts5_aux.c \
+   $(TOP)/ext/fts5/fts5_buffer.c \
+   $(TOP)/ext/fts5/fts5.c \
+   $(TOP)/ext/fts5/fts5_config.c \
+   $(TOP)/ext/fts5/fts5_expr.c \
+   $(TOP)/ext/fts5/fts5_index.c \
+   fts5parse.c \
+   $(TOP)/ext/fts5/fts5_storage.c 
+
 
 # Generated source code files
 #
@@ -375,6 +397,9 @@ EXTHDR += \
   $(TOP)/ext/rtree/rtree.h
 EXTHDR += \
   $(TOP)/ext/icu/sqliteicu.h
+EXTHDR += \
+  $(TOP)/ext/fts5/fts5Int.h  \
+  $(TOP)/ext/fts5/fts5.h 
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -553,10 +578,39 @@ fts3_unicode2.o:	$(TOP)/ext/fts3/fts3_unicode2.c $(HDR) $(EXTHDR)
 fts3_write.o:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
 
+fts5.o:	$(TOP)/ext/fts5/fts5.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5.c
+
 rtree.o:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
 
 
+# FTS5 things
+#
+fts5_aux.o:	$(TOP)/ext/fts5/fts5_aux.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_aux.c
+
+fts5_buffer.o:	$(TOP)/ext/fts5/fts5_buffer.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_buffer.c
+
+fts5_config.o:	$(TOP)/ext/fts5/fts5_config.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_config.c
+
+fts5_expr.o:	$(TOP)/ext/fts5/fts5_expr.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_expr.c
+
+fts5_index.o:	$(TOP)/ext/fts5/fts5_index.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_index.c
+
+fts5_storage.o:	$(TOP)/ext/fts5/fts5_storage.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c
+
+fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
+	cp $(TOP)/ext/fts5/fts5parse.y .
+	rm -f fts5parse.h
+	./lemon $(OPTS) fts5parse.y
+
+
 # Rules for building test programs and for running tests
 #
 tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a
@@ -642,6 +696,9 @@ wordcount$(EXE):	$(TOP)/test/wordcount.c sqlite3.c
 speedtest1$(EXE):	$(TOP)/test/speedtest1.c sqlite3.o
 	$(TCC) -I. -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB)
 
+loadfts: $(TOP)/tool/loadfts.c libsqlite3.a
+	$(TCC) $(TOP)/tool/loadfts.c libsqlite3.a -o loadfts $(THREADLIB)
+
 # This target will fail if the SQLite amalgamation contains any exported
 # symbols that do not begin with "sqlite3_". It is run as part of the
 # releasetest.tcl script.

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\sSQLITE_OMIT_WSD\sbuilds.
-D 2014-06-23T10:18:50.447
+C Try\sto\sreuse\ssorter\sstatements\sin\sfts5.\sDoes\snot\swork\sdue\sto\scircular\sreferences\son\sVTable\sobject.
+D 2014-07-31T17:47:04.870
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -79,7 +79,7 @@ F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
 F ext/fts3/fts3.c 20bc65862cfcea0a39bb64a819f8fe92a8e144c1
-F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
+F ext/fts3/fts3.h 62a77d880cf06a2865052726f8325c8fabcecad7
 F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918
 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365
 F ext/fts3/fts3_expr.c 351395fad6fcb16ecfc61db0861008a70101330c
@@ -103,6 +103,16 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
+F ext/fts5/fts5.c 85b0d66a927ce160ca226b121cd5ae24affa5b85
+F ext/fts5/fts5.h 8ace10d5b249a3baa983c79e7a1306d2a79cfd6a
+F ext/fts5/fts5Int.h 2adf0ca39d9764cb2114725f5a1c8c7a0fdcd698
+F ext/fts5/fts5_aux.c 366057c7186bc3615deb5ecc0ff61de50b6d2dbc
+F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
+F ext/fts5/fts5_config.c f4ebf143e141b8c77355e3b15aba81b7be51d710
+F ext/fts5/fts5_expr.c e764d75c58a3accda795f1da1b45960ac87dc77a
+F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997
+F ext/fts5/fts5_storage.c 7036cf22baa00780860fc35cc2573d36cdc03f7b
+F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -146,7 +156,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 7b1d0be0840f213405c977c87917241158126a33
+F main.mk 8118631727a27fa88eb38a07ac3b86ecb86e9eb0
 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
 F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -189,7 +199,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12
 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
 F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303
-F src/main.c 7c2c3cafdd6313c8f9319ebec1565782e624372e
+F src/main.c e777879ad7c431f5b3b5d49c8419727b61d7c1be
 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b
@@ -585,6 +595,14 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
+F test/fts5aa.test f90836c002804a82386d66c79b380128c5f3005e
+F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
+F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
+F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
+F test/fts5ae.test cb37b3135a00d3afd5492ec534ecf654be5ff69e
+F test/fts5af.test 9ebe23aa3875896076952c7bc6e8308813a63c74
+F test/fts5ag.test 0747bf3bade16d5165810cf891f875933b28b420
+F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
@@ -751,7 +769,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
-F test/permutations.test bc474bafb022cc5014ef3a9c3d5ab61d6d6f587c
+F test/permutations.test 5f1f942bae4139b33626b82627aa262c0f72d936
 F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@@ -1141,6 +1159,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
+F tool/loadfts.c 3bdd46090112c84df44a4fbae740af3836108b3f
 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
 F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
@@ -1148,7 +1167,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d
-F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6
+F tool/mksqlite3c.tcl becaa9d5617dfe137e73dddda9dab8f58bc71e8c
 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12
 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1
 F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105
@@ -1179,7 +1198,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 612b6d1b1f74eaf618520b90811eca10f978fc71
-R 283ec0802d51fd4e82222d529f6a8475
+P 37a417d27e4ebafd4783f62728d7467316b75b17
+R 534b5c94a0bdd15a21f4d009040d2019
+T *branch * save_sorter_stmt
+T *sym-save_sorter_stmt *
+T -sym-fts5 *
 U dan
-Z de359222916ca6f6bd684ca986937509
+Z b48f3961f977602946001c5dfc0bf2c0

manifest.uuid

@@ -1 +1 @@
-07dda49c1bf8997a18c3368acb81b6d863ea38d6
+bc14e64bdffb165fc5468339a7eaa5f6707537bf

src/main.c

@@ -2609,6 +2609,7 @@ static int openDatabase(
 #ifdef SQLITE_ENABLE_FTS3
   if( !db->mallocFailed && rc==SQLITE_OK ){
     rc = sqlite3Fts3Init(db);
+    if( rc==SQLITE_OK ) rc = sqlite3Fts5Init(db);
   }
 #endif
 

test/fts5aa.test

@@ -0,0 +1,289 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5aa
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
+  SELECT name, sql FROM sqlite_master;
+} {
+  t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
+  t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
+  t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
+  t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
+}
+
+do_execsql_test 1.1 {
+  DROP TABLE t1;
+  SELECT name, sql FROM sqlite_master;
+} {
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+}
+do_execsql_test 2.1 {
+  INSERT INTO t1 VALUES('a b c', 'd e f');
+}
+do_execsql_test 2.2 {
+  SELECT fts5_decode(id, block) FROM t1_data WHERE id==10
+} {
+  {{structure idx=0} {lvl=0 nMerge=0 {id=27723 h=1 leaves=1..1}}}
+}
+do_execsql_test 2.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+}
+foreach {i x y} {
+   1  {g f d b f} {h h e i a}
+   2  {f i g j e} {i j c f f}
+   3  {e e i f a} {e h f d f}
+   4  {h j f j i} {h a c f j}
+   5  {d b j c g} {f e i b e}
+   6  {a j a e e} {j d f d e}
+   7  {g i j c h} {j d h c a}
+   8  {j j i d d} {e e d f b}
+   9  {c j j d c} {h j i f g}
+   10 {b f h i a} {c f b b j}
+} {
+  do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
+  do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  if {[set_test_counter errors]} break
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+foreach {i x y} {
+   1  {g f d b f} {h h e i a}
+   2  {f i g j e} {i j c f f}
+   3  {e e i f a} {e h f d f}
+   4  {h j f j i} {h a c f j}
+   5  {d b j c g} {f e i b e}
+   6  {a j a e e} {j d f d e}
+   7  {g i j c h} {j d h c a}
+   8  {j j i d d} {e e d f b}
+   9  {c j j d c} {h j i f g}
+   10 {b f h i a} {c f b b j}
+} {
+  do_execsql_test 4.$i.1 { INSERT INTO t1 VALUES($x, $y) }
+  do_execsql_test 4.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  if {[set_test_counter errors]} break
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+foreach {i x y} {
+   1  {dd abc abc abc abcde} {aaa dd ddd ddd aab}
+   2  {dd aab d aaa b} {abcde c aaa aaa aaa}
+   3  {abcde dd b b dd} {abc abc d abc ddddd}
+   4  {aaa abcde dddd dddd abcde} {abc b b abcde abc}
+   5  {aab dddd d dddd c} {ddd abcde dddd abcde c}
+   6  {ddd dd b aab abcde} {d ddddd dddd c abc}
+   7  {d ddddd ddd c abcde} {c aab d abcde ddd}
+   8  {abcde aaa aab c c} {ddd c dddd b aaa}
+   9  {abcde aab ddddd c aab} {dddd dddd b c dd}
+   10 {ddd abcde dddd dd c} {dddd c c d abcde}
+} {
+  do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
+  do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  if {[set_test_counter errors]} break
+}
+
+#-------------------------------------------------------------------------
+#
+breakpoint
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+
+do_execsql_test 6.1 {
+  INSERT  INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
+  REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
+}
+
+do_execsql_test 6.2 {
+  INSERT INTO t1(t1) VALUES('integrity-check') 
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+
+proc doc {} {
+  set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
+  set ret [list]
+  for {set j 0} {$j < 20} {incr j} {
+    lappend ret [lindex $v [expr int(rand()*[llength $v])]]
+  }
+  return $ret
+}
+
+proc dump_structure {} {
+  db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
+    foreach lvl [lrange $t 1 end] {
+      set seg [string repeat . [expr [llength $lvl]-2]]
+      puts "[lrange $lvl 0 1] $seg"
+    }
+  }
+}
+
+for {set i 1} {$i <= 10} {incr i} {
+  do_test 7.$i {
+    for {set j 0} {$j < 100} {incr j} {
+      set x [doc]
+      set y [doc]
+      set z [doc]
+      set rowid [expr int(rand() * 100)]
+      execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
+    }
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
+  } {}
+  if {[set_test_counter errors]} exit
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+
+do_execsql_test 8.1 {
+  INSERT INTO t1 VALUES('the quick brown fox');
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+
+#-------------------------------------------------------------------------
+#
+reset_db
+
+expr srand(0)
+
+do_execsql_test 9.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+
+proc doc {} {
+  set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
+  set ret [list]
+  for {set j 0} {$j < 20} {incr j} {
+    lappend ret [lindex $v [expr int(rand()*[llength $v])]]
+  }
+  return $ret
+}
+
+proc dump_structure {} {
+  db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
+    foreach lvl [lrange $t 1 end] {
+      set seg [string repeat . [expr [llength $lvl]-2]]
+      puts "[lrange $lvl 0 1] $seg"
+    }
+  }
+}
+
+for {set i 1} {$i <= 10} {incr i} {
+  do_test 9.$i {
+    for {set j 0} {$j < 100} {incr j} {
+      set x [doc]
+      set y [doc]
+      set z [doc]
+      set rowid [expr int(rand() * 100)]
+      execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
+    }
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
+  } {}
+  if {[set_test_counter errors]} break
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 10.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
+}
+set d10 {
+   1  {g f d b f} {h h e i a}
+   2  {f i g j e} {i j c f f}
+   3  {e e i f a} {e h f d f}
+   4  {h j f j i} {h a c f j}
+   5  {d b j c g} {f e i b e}
+   6  {a j a e e} {j d f d e}
+   7  {g i j c h} {j d h c a}
+   8  {j j i d d} {e e d f b}
+   9  {c j j d c} {h j i f g}
+  10  {b f h i a} {c f b b j}
+}
+foreach {rowid x y} $d10 {
+  do_execsql_test 10.1.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
+  do_execsql_test 10.1.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+}
+foreach rowid {5 9 8 1 2 4 10 7 3 5 6} {
+  do_execsql_test 10.2.$rowid.1 { DELETE FROM t1 WHERE rowid = $rowid }
+  do_execsql_test 10.2.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+}
+foreach {rowid x y} $d10 {
+  do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
+  do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+}
+
+do_execsql_test 10.4.1 { DELETE FROM t1 }
+do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+
+#-------------------------------------------------------------------------
+#
+do_catchsql_test 11.1 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
+} {1 {reserved fts5 column name: rank}}
+do_catchsql_test 11.2 {
+  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
+} {1 {reserved fts5 table name: rank}}
+
+finish_test
+

test/fts5ab.test

@@ -0,0 +1,154 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ab
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  INSERT INTO t1 VALUES('hello', 'world');
+  INSERT INTO t1 VALUES('one two', 'three four');
+  INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five');
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1;
+} { forty five {one two} {three four} hello world }
+
+do_execsql_test 1.2 {
+  SELECT rowid FROM t1;
+} {45 2 1}
+
+do_execsql_test 1.3 {
+  SELECT rowid FROM t1 ORDER BY rowid ASC;
+} {1 2 45}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1 WHERE rowid=2;
+} {{one two} {three four}}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t1 WHERE rowid=2.01;
+} {}
+
+do_execsql_test 1.6 {
+  SELECT * FROM t1 WHERE rowid=1.99;
+} {}
+
+#-------------------------------------------------------------------------
+
+reset_db
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1 VALUES('one');
+  INSERT INTO t1 VALUES('two');
+  INSERT INTO t1 VALUES('three');
+}
+
+do_catchsql_test 2.2 {
+  SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'
+} {1 {fts5: syntax error near "AND"}}
+
+do_execsql_test 2.3 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'two' } {2 two}
+do_execsql_test 2.4 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'three' } {3 three}
+do_execsql_test 2.5 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'one' } {1 one}
+
+do_execsql_test 2.6 {
+  INSERT INTO t1 VALUES('a b c d e f g');
+  INSERT INTO t1 VALUES('b d e a a a i');
+  INSERT INTO t1 VALUES('x y z b c c c');
+}
+
+foreach {tn expr res} {
+  1  a    {5 4}
+  2  b    {6 5 4}
+  3  c    {6 4}
+  4  d    {5 4}
+  5  e    {5 4}
+  6  f    {4}
+  7  g    {4}
+  8  x    {6}
+  9  y    {6}
+  10 z    {6}
+} {
+  do_execsql_test 2.7.$tn { SELECT rowid FROM t1 WHERE t1 MATCH $expr } $res
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a,b);
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+
+foreach {tn a b} {
+   1 {abashed abandons abase abash abaft} {abases abased}
+   2 {abasing abases abaft abated abandons} {abases abandoned}
+   3 {abatement abash abash abated abase} {abasements abashing}
+   4 {abaft abasements abase abasement abasing} {abasement abases}
+   5 {abaft abashing abatement abash abasements} {abandons abandoning}
+   6 {aback abate abasements abashes abandoned} {abasement abased}
+   7 {abandons abated abased aback abandoning} {abases abandoned}
+   8 {abashing abases abasement abaft abashing} {abashed abate}
+   9 {abash abase abate abashing abashed} {abandon abandoned}
+   10 {abate abandoning abandons abasement aback} {abandon abandoning}
+} {
+  do_execsql_test 3.1.$tn.1 { INSERT INTO t1 VALUES($a, $b) } 
+  do_execsql_test 3.1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+}
+
+foreach {tn expr res} {
+  1 {abash} {9 5 3 1}
+  2 {abase} {9 4 3 1}
+  3 {abase + abash} {1}
+  4 {abash + abase} {9}
+  5 {abaft + abashing} {8 5}
+  6 {abandon + abandoning} {10}
+  7 {"abashing abases abasement abaft abashing"} {8}
+} {
+  do_execsql_test 3.2.$tn {
+    SELECT rowid FROM t1 WHERE t1 MATCH $expr
+  } $res
+}
+
+do_execsql_test 3.3 {
+  SELECT rowid FROM t1 WHERE t1 MATCH 'NEAR(aback abate, 2)'
+} {6}
+
+foreach {tn expr res} {
+  1 {abash} {1 3 5 9}
+  2 {abase} {1 3 4 9}
+  3 {abase + abash} {1}
+  4 {abash + abase} {9}
+  5 {abaft + abashing} {5 8}
+  6 {abandon + abandoning} {10}
+  7 {"abashing abases abasement abaft abashing"} {8}
+} {
+  do_execsql_test 3.4.$tn {
+    SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC
+  } $res
+}
+
+
+finish_test

test/fts5ac.test

@@ -0,0 +1,400 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ac
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE xx USING fts5(x,y);
+  INSERT INTO xx(xx) VALUES('pgsz=32');
+}
+
+set data {
+    0   {p o q e z k z p n f y u z y n y}   {l o o l v v k}
+    1   {p k h h p y l l h i p v n}         {p p l u r i f a j g e r r x w}
+    2   {l s z j k i m p s}                 {l w e j t j e e i t w r o p o}
+    3   {x g y m y m h p}                   {k j j b r e y y a k y}
+    4   {q m a i y i z}                     {o w a g k x g j m w e u k}
+    5   {k o a w y b s z}                   {s g l m m l m g p}
+    6   {d a q i z h b l c p k j g k}       {p x u j x t v c z}
+    7   {f d a g o c t i}                   {w f c x l d r k i j}
+    8   {y g w u b q p o m j y b p a e k}   {r i d k y w o z q m a t p}
+    9   {r k o m c c j s x m x m x m q r}   {y r c a q d z k n x n}
+    10  {k j q m g q a j d}                 {d d e z g w h c d o o g x d}
+    11  {j z u m o y q j f w e e w t r j w} {g m o r x n t n w i f g l z f}
+    12  {s y w a w d o h x m k}             {c w k z b p o r a}
+    13  {u t h x e g s k n g i}             {f j w g c s r}
+    14  {b f i c s u z t k}                 {c k q s j u i z o}
+    15  {n a f n u s w h y n s i q e w}     {x g e g a s s h n}
+    16  {k s q e j n p}                     {t r j f t o e k k l m i}
+    17  {g d t u w r o p m n m n p h b o u} {h s w o s l j e}
+    18  {f l q y q q g e e x j r}           {n b r r g e i r t x q k}
+    19  {f i r g o a w e p i l o a w}       {e k r z t d g h g i b d i e m}
+    20  {l d u u f p y}                     {g o m m u x m g l j t t x x u}
+    21  {m c d k x i c z l}                 {m i a i e u h}
+    22  {w b f o c g x y j}                 {z d w x d f h i p}
+    23  {w u i u x t c h k i b}             {b y k h b v r t g j}
+    24  {h f d j s w s b a p k}             {a q y u z e y m m j q r}
+    25  {d i x y x x k i y f s d j h z p n} {l l q m e t c w g y h t s v g}
+    26  {g s q w t d k x g f m j p k y}     {r m b x e l t d}
+    27  {j l s q u g y v e c l o}           {m f l m m m h g x x l n c}
+    28  {c t j g v r s b z j}               {l c f y d t q n}
+    29  {e x z y w i h l}                   {b n b x e y q e n u m}
+    30  {g y y h j b w r}                   {q b q f u s k c k g r}
+    31  {g u l x l b r c m z b u c}         {k g t b x k x n t e z d h o}
+    32  {w g v l z f b z h p s c v h}       {g e w v m h k r g w a r f q}
+    33  {c g n f u d o y o b}               {e y o h x x y y i z s b h a j}
+    34  {v y h c q u u s q y x x k s q}     {d n r m y k n t i r n w e}
+    35  {o u c x l e b t a}                 {y b a x y f z x r}
+    36  {x p h l j a a u u j h}             {x o f s z m b c q p}
+    37  {k q t i c a q n m v v}             {v r z e f m y o}
+    38  {r w t t t t r v v o e p g h}       {l w x a g a u h y}
+    39  {o p v g v b a g o}                 {j t q c r b b g y z}
+    40  {f s o r o d t h q f x l}           {r d b m k i f s t d l m y x j w}
+    41  {t m o t m f m f}                   {i p i q j v n v m b q}
+    42  {t x w a r l w d t b c o d o}       {a h f h w z d n s}
+    43  {t u q c d g p q x j o l c x c}     {m n t o z z j a y}
+    44  {v d i i k b f s z r v r z y}       {g n q y s x x m b x c l w}
+    45  {p v v a c s z y e o l}             {m v t u d k m k q b d c v z r}
+    46  {f y k l d r q w r s t r e}         {h m v r r l r r t f q e x y}
+    47  {w l n l t y x}                     {n h s l a f c h u f l x x m v n o}
+    48  {t n v i k e b p z p d j j l i o}   {i v z p g u e j s i k n h w d c}
+    49  {z v x p n l t a j c}               {e j l e n c e t a d}
+    50  {w u b x u i v h a i y m m r p m s} {s r h d o g z y f f x e}
+    51  {d c c x b c a x g}                 {p r a j v u y}
+    52  {f w g r c o d l t u e z h i}       {j l l s s b j m}
+    53  {p m t f k i x}                     {u v y a z g w v v m x h i}
+    54  {l c z g l o j i c d e b}           {b f v y w u i b e i y}
+    55  {r h c x f x a d s}                 {z x y k f l r b q c v}
+    56  {v x x c y h z x b g m o q n c}     {h n b i t g h a q b c o r u}
+    57  {d g l o h t b s b r}               {n u e p t i m u}
+    58  {t d y e t d c w u o s w x f c h}   {i o s v y b r d r}
+    59  {l b a p q n d r}                   {k d c c d n y q h g a o p e x}
+    60  {f r z v m p k r}                   {x x r i s b a g f c}
+    61  {s a z i e r f i w c n y v z t k s} {y y i r y n l s b w i e k n}
+    62  {n x p r e x q r m v i b y}         {f o o z n b s r q j}
+    63  {y j s u j x o n r q t f}           {f v k n v x u s o a d e f e}
+    64  {u s i l y c x q}                   {r k c h p c h b o s s u s p b}
+    65  {m p i o s h o}                     {s w h u n d m n q t y k b w c}
+    66  {l d f g m x x x o}                 {s w d d f b y j j h h t i y p j o}
+    67  {c b m h f n v w n h}               {i r w i e x r w l z p x u g u l s}
+    68  {y a h u h i m a y q}               {d d r x h e v q n z y c j}
+    69  {c x f d x o n p o b r t b l p l}   {m i t k b x v f p t m l l y r o}
+    70  {u t l w w m s}                     {m f m o l t k o p e}
+    71  {f g q e l n d m z x q}             {z s i i i m f w w f n g p e q}
+    72  {n l h a v u o d f j d e x}         {v v s l f g d g r a j x i f z x}
+    73  {x v m v f i g q e w}               {r y s j i k m j j e d g r n o i f}
+    74  {g d y n o h p s y q z j d w n h w} {x o d l t j i b r d o r y}
+    75  {p g b i u r b e q d v o a g w m k} {q y z s f q o h}
+    76  {u z a q u f i f f b}               {b s p b a a d x r r i q f}
+    77  {w h h z t h p o a h h e e}         {h w r p h k z v y f r x}
+    78  {c a r k i a p u x}                 {f w l p t e m l}
+    79  {q q u k o t r k z}                 {f b m c w p s s o z}
+    80  {t i g v y q s r x m r x z e f}     {x o j w a u e y s j c b u p p r o}
+    81  {n j n h r l a r e o z w e}         {v o r r j a v b}
+    82  {i f i d k w d n h}                 {o i d z i z l m w s b q v u}
+    83  {m d g q q b k b w f q q p p}       {j m q f b y c i z k y q p l e a}
+    84  {m x o n y f g}                     {y c n x n q j i y c l h b r q z}
+    85  {v o z l n p c}                     {g n j n t b b x n c l d a g j v}
+    86  {z n a y f b t k k t d b z a v}     {r p c n r u k u}
+    87  {b q t x z e c w}                   {q a o a l o a h i m j r}
+    88  {j f h o x x a z g b a f a m i b}   {j z c z y x e x w t}
+    89  {t c t p r s u c q n}               {z x l i k n f q l n t}
+    90  {w t d q j g m r f k n}             {l e w f w w a l y q k i q t p c t}
+    91  {c b o k l i c b s j n m b l}       {y f p q o w g}
+    92  {f y d j o q t c c q m f j s t}     {f h e d y m o k}
+    93  {k x j r m a d o i z j}             {r t t t f e b r x i v j v g o}
+    94  {s f e a e t i h h d q p z t q}     {b k m k w h c}
+    95  {h b n j t k i h o q u}             {w n g i t o k c a m y p f l x c p}
+    96  {f c x p y r b m o l m o a}         {p c a q s u n n x d c f a o}
+    97  {u h h k m n k}                     {u b v n u a o c}
+    98  {s p e t c z d f n w f}             {l s f j b l c e s h}
+    99  {r c v w i v h a t a c v c r e}     {h h u m g o f b a e o}
+}
+
+do_test 1.1 {
+  foreach {id x y} $data {
+    execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
+  }
+} {}
+
+# Usage:
+#
+#   poslist aCol ?-near N? ?-col C? -- phrase1 phrase2...
+#
+proc poslist {aCol args} {
+  set O(-near) 10
+  set O(-col)  -1
+
+  set nOpt [lsearch -exact $args --]
+  if {$nOpt<0} { error "no -- option" }
+
+  foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
+    if {[info exists O($k)]==0} { error "unrecognized option $k" }
+    set O($k) $v
+  }
+
+  # Set $phraselist to be a list of phrases. $nPhrase its length.
+  set phraselist [lrange $args [expr $nOpt+1] end]
+  set nPhrase [llength $phraselist]
+
+  for {set j 0} {$j < [llength $aCol]} {incr j} {
+    for {set i 0} {$i < $nPhrase} {incr i} { 
+      set A($j,$i) [list]
+    }
+  }
+
+  set iCol -1
+  foreach col $aCol {
+    incr iCol
+    if {$O(-col)>=0 && $O(-col)!=$iCol} continue
+
+    set nToken [llength $col]
+
+    set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
+    for { } {$iFL < $nToken} {incr iFL} {
+      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
+        set B($iPhrase) [list]
+      }
+      
+      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
+        set p [lindex $phraselist $iPhrase]
+        set nPm1 [expr {[llength $p] - 1}]
+        set iFirst [expr $iFL - $O(-near) - [llength $p]]
+
+        for {set i $iFirst} {$i <= $iFL} {incr i} {
+          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
+        }
+        if {[llength $B($iPhrase)] == 0} break
+      }
+
+      if {$iPhrase==$nPhrase} {
+        for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
+          set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
+          set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
+        }
+      }
+    }
+  }
+
+  set res [list]
+  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
+    set plist [list]
+    for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
+      foreach a $A($iCol,$iPhrase) {
+        lappend plist "$iCol.$a"
+      }
+    }
+    lappend res $plist
+  }
+
+    #puts $res
+  return $res
+}
+
+# Usage:
+#
+#   nearset aCol ?-near N? ?-col C? -- phrase1 phrase2...
+#
+proc nearset {args} {
+  set plist [poslist {*}$args]
+  return [expr [llength [lindex $plist 0]]>0]
+}
+
+# Argument $expr is an FTS5 match expression designed to be executed against
+# an FTS5 table with the following schema:
+# 
+#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
+#
+# Assuming the table contains the same records as stored int the global 
+# $::data array (see above), this function returns a list containing one
+# element for each match in the dataset. The elements are themselves lists
+# formatted as follows:
+#
+#   <rowid> {<phrase 0 matches> <phrase 1 matches>...}
+#
+# where each <phrase X matches> element is a list of phrase matches in the
+# same form as returned by auxiliary scalar function fts5_test().
+#
+proc matchdata {bPos expr {bAsc 0}} {
+
+  set tclexpr [db one {SELECT fts5_expr_tcl($expr, 'nearset $cols', 'x', 'y')}]
+  set res [list]
+
+  #puts $tclexpr
+  foreach {id x y} $::data {
+    set cols [list $x $y]
+    if $tclexpr {
+      if {$bPos} {
+        set N [regexp -all -inline {\[nearset [^\]]*\]} $tclexpr]
+        set rowres [list]
+        foreach phrase $N {
+          set cmd "poslist [string range $phrase 9 end-1]"
+          set pos [eval $cmd]
+          set rowres [concat $rowres $pos]
+        }
+        lappend res [list $id $rowres]
+      } else {
+        lappend res $id
+      }
+    }
+  }
+
+  if {$bAsc} {
+    set res [lsort -integer -increasing -index 0 $res]
+  } else {
+    set res [lsort -integer -decreasing -index 0 $res]
+  }
+
+  return [concat {*}$res]
+}
+
+#
+# End of test code
+#-------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------
+# Test phrase queries.
+#
+foreach {tn phrase} {
+  1 "o"
+  2 "b q"
+  3 "e a e"
+  4 "m d g q q b k b w f q q p p"
+  5 "l o o l v v k"
+  6 "a"
+  7 "b"
+  8 "c"
+  9 "no"
+  10 "L O O L V V K"
+} {
+  set expr "\"$phrase\""
+  set res [matchdata 1 $expr]
+
+  do_execsql_test 1.2.$tn.[llength $res] { 
+    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+  } $res
+}
+
+#-------------------------------------------------------------------------
+# Test some AND and OR queries.
+#
+foreach {tn expr} {
+  1.1 "a   AND b"
+  1.2 "a+b AND c"
+  1.3 "d+c AND u"
+  1.4 "d+c AND u+d"
+
+  2.1 "a   OR b"
+  2.2 "a+b OR c"
+  2.3 "d+c OR u"
+  2.4 "d+c OR u+d"
+
+  3.1 { a AND b AND c }
+} {
+  set res [matchdata 1 $expr]
+  do_execsql_test 2.$tn.[llength $res] { 
+    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+  } $res
+}
+
+#-------------------------------------------------------------------------
+# Queries on a specific column.
+#
+foreach {tn expr} {
+  1 "x:a"
+  2 "y:a"
+  3 "x:b"
+  4 "y:b"
+} {
+  set res [matchdata 1 $expr]
+  do_execsql_test 3.$tn.[llength $res] { 
+    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+  } $res
+}
+
+#-------------------------------------------------------------------------
+# Some NEAR queries.
+#
+foreach {tn expr} {
+  1 "NEAR(a b)"
+  2 "NEAR(r c)"
+  2 { NEAR(r c, 5) }
+  3 { NEAR(r c, 3) }
+  4 { NEAR(r c, 2) }
+  5 { NEAR(r c, 0) }
+  6 { NEAR(a b c) }
+  7 { NEAR(a b c, 8) }
+  8  { x : NEAR(r c) }
+  9  { y : NEAR(r c) }
+} {
+  set res [matchdata 1 $expr]
+  do_execsql_test 4.1.$tn.[llength $res] { 
+    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+  } $res
+}
+
+do_test 4.1  { poslist {{a b c}} -- a } {0.0}
+do_test 4.2  { poslist {{a b c}} -- c } {0.2}
+
+foreach {tn expr tclexpr} {
+  1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
+} {
+  do_execsql_test 5.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr]
+}
+
+#-------------------------------------------------------------------------
+#
+foreach {bAsc sql} {
+  0 {SELECT rowid FROM xx WHERE xx MATCH $expr}
+  1 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid ASC}
+} {
+  foreach {tn expr} {
+    0.1 x
+    1 { NEAR(r c) }
+    2 { NEAR(r c, 5) }
+    3 { NEAR(r c, 3) }
+    4 { NEAR(r c, 2) }
+    5 { NEAR(r c, 0) }
+    6 { NEAR(a b c) }
+    7 { NEAR(a b c, 8) }
+    8  { x : NEAR(r c) }
+    9  { y : NEAR(r c) }
+    10 { x : "r c" }
+    11 { y : "r c" }
+    12 { a AND b }
+    13 { a AND b AND c }
+    14a { a }
+    14b { a OR b }
+    15 { a OR b AND c }
+    16 { c AND b OR a }
+    17 { c AND (b OR a) }
+    18 { c NOT (b OR a) }
+    19 { c NOT b OR a AND d }
+  } {
+    set res [matchdata 0 $expr $bAsc]
+    do_execsql_test 6.$bAsc.$tn.[llength $res] $sql $res
+  }
+}
+
+finish_test
+

test/fts5ad.test

@@ -0,0 +1,215 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ad
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE yy USING fts5(x, y);
+  INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching');
+  INSERT INTO yy VALUES('indices (or all  matching', 'values if -inline is');
+  INSERT INTO yy VALUES('specified as  well.) If', 'indices are returned, the');
+} {}
+
+foreach {tn match res} {
+  1 {c*} {1}
+  2 {i*} {3 2}
+  3 {t*} {3 1}
+  4 {r*} {3 1}
+} {
+  do_execsql_test 1.$tn {
+    SELECT rowid FROM yy WHERE yy MATCH $match
+  } $res
+}
+
+foreach {tn match res} {
+  5 {c*} {1}
+  6 {i*} {2 3}
+  7 {t*} {1 3}
+  8 {r*} {1 3}
+} {
+  do_execsql_test 1.$tn {
+    SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC
+  } $res
+}
+
+foreach {T create} {
+  2 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+    INSERT INTO t1(t1) VALUES('pgsz=32');
+  }
+  
+  3 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
+    INSERT INTO t1(t1) VALUES('pgsz=32');
+  }
+
+} {
+
+  do_test $T.1 { 
+    execsql { DROP TABLE IF EXISTS t1 }
+    execsql $create
+  } {}
+  
+  do_test $T.1 {
+    foreach {rowid a b} {
+      0   {fghij uvwxyz klmn pq uvwx}         {klmn f fgh uv fghij klmno}
+      1   {uv f abcd abcd fghi}               {pq klm uv uv fgh uv a}
+      2   {klmn klm pqrs fghij uv}            {f k uvw ab abcd pqr uv}
+      3   {ab pqrst a fghi ab pqr fg}         {k klmno a fg abcd}
+      4   {abcd pqrst uvwx a fgh}             {f klmno fghij kl pqrst}
+      5   {uvwxyz k abcde u a}                {uv k k kl klmn}
+      6   {uvwxyz k klmn pqrst uv}            {fghi pqrs abcde u k}
+      7   {uvwxy klmn u p pqrst fgh}          {p f fghi abcd uvw kl uv}
+      8   {f klmno pqrst uvwxy pqrst}         {uv abcde klm pq pqr}
+      9   {f abcde a uvwxyz pqrst}            {fghij abc k uvwx pqr fghij uvwxy}
+      10  {ab uv f fg pqrst uvwxy}            {fgh p uv k abc klm uvw}
+      11  {pq klmno a uvw abcde uvwxyz}       {fghij pq uvwxyz pqr fghi}
+      12  {fgh u pq fgh uvw}                  {uvw pqr f uvwxy uvwx}
+      13  {uvwx klmn f fgh abcd pqr}          {uvw k fg uv klm abcd}
+      14  {ab uvwx pqrst pqr uvwxyz pqrs}     {uvwxyz abcde ab ab uvw abcde}
+      15  {abc abcde uvwxyz abc kl k pqr}     {klm k k klmno u fgh}
+      16  {fghi abcd fghij uv uvwxyz ab uv}   {klmn pqr a uvw fghi}
+      17  {abc pqrst fghi uvwx uvw klmn fghi} {ab fg pqr pqrs p}
+      18  {pqr kl a fghij fgh fg kl}          {pqr uvwxyz uvw abcd uvwxyz}
+      19  {fghi fghi pqr kl fghi f}           {klmn u u klmno klmno}
+      20  {abc pqrst klmno kl pq uvwxy}       {abc k fghi pqrs klm}
+      21  {a pqr uvwxyz uv fghi a fgh}        {abc pqrs pqrst pq klm}
+      22  {klm abc uvwxyz klm pqrst}          {fghij k pq pqr u klm fghij}
+      23  {p klm uv p a a}                    {uvwxy klmn uvw abcde pq}
+      24  {uv fgh fg pq uvwxy u uvwxy}        {pqrs a uvw p uvwx uvwxyz fg}
+      25  {fghij fghi klmn abcd pq kl}        {fghi abcde pqrs abcd fgh uvwxy}
+      26  {pq fgh a abc klmno klmn}           {fgh p k p fg fghij}
+      27  {fg pq kl uvwx fghij pqrst klmn}    {abcd uvw abcd fghij f fghij}
+      28  {uvw fghi p fghij pq fgh uvwx}      {k fghij abcd uvwx pqr fghi}
+      29  {klm pq abcd pq f uvwxy}            {pqrst p fghij pqr p}
+      30  {ab uvwx fg uvwx klmn klm}          {klmn klmno fghij klmn klm}
+      31  {pq k pqr abcd a pqrs}              {abcd abcd uvw a abcd klmno ab}
+      32  {pqrst u abc pq klm}                {abc kl uvwxyz fghij u fghi p}
+      33  {f uvwxy u k f uvw uvwx}            {pqrs uvw fghi fg pqrst klm}
+      34  {pqrs pq fghij uvwxyz pqr}          {ab abc abc uvw f pq f}
+      35  {uvwxy ab uvwxy klmno kl pqrs}      {abcde uvw pqrs uvwx k k}
+      36  {uvwxyz k ab abcde abc uvw}         {uvw abcde uvw klmn uv klmn}
+      37  {k kl uv abcde uvwx fg u}           {u abc uvwxy k fg abcd}
+      38  {fghi pqrst fghi pqr pqrst uvwx}    {u uv uvwx fghi abcde}
+      39  {k pqrst k uvw fg pqrst fghij}      {uvwxy ab kl klmn uvwxyz abcde}
+      40  {fg uvwxy pqrs klmn uvwxyz klm p}   {k uv ab fghij fgh k pqrs}
+      41  {uvwx abc f pq uvwxy k}             {ab uvwxyz abc f fghij}
+      42  {uvwxy klmno uvwxyz uvwxyz pqrst}   {uv kl kl klmno k f abcde}
+      43  {abcde ab pqrs fg f fgh}            {abc fghij fghi k k}
+      44  {uvw abcd a ab pqrst klmn fg}       {pqrst u uvwx pqrst fghij f pqrst}
+      45  {uvwxy p kl uvwxyz ab pqrst fghi}   {abc f pqr fg a k}
+      46  {u p f a fgh}                       {a kl pq uv f}
+      47  {pqrs abc fghij fg abcde ab a}      {p ab uv pqrs kl fghi abcd}
+      48  {abcde uvwxy pqrst uv abc pqr uvwx} {uvwxy klm uvwxy uvwx k}
+      49  {fgh klm abcde klmno u}             {a f fghij f uvwxyz abc u}
+      50  {uv uvw uvwxyz uvwxyz uv ab}        {uvwx pq fg u k uvwxy}
+      51  {uvwxy pq p kl fghi}                {pqrs fghi pqrs abcde uvwxyz ab}
+      52  {pqr p uvwxy kl pqrs klmno fghij}   {ab abcde abc pqrst pqrs uv}
+      53  {fgh pqrst p a klmno}               {ab ab pqrst pqr kl pqrst}
+      54  {abcd klm ab uvw a fg u}            {f pqr f abcd uv}
+      55  {u fg uvwxyz k uvw}                 {abc pqrs f fghij fg pqrs uvwxy}
+      56  {klm fg p fghi fg a}                {uv a fghi uvwxyz a fghi}
+      57  {uvwxy k abcde fgh f fghi}          {f kl klmn f fghi klm}
+      58  {klm k fgh uvw fgh fghi}            {klmno uvwx u pqrst u}
+      59  {fghi pqr pqrst p uvw fghij}        {uv pqrst pqrs pq fghij klm}
+      60  {uvwx klm uvwxy uv klmn}            {p a a abc klmn ab k}
+      61  {uvwxy uvwx klm uvwx klm}           {pqrs ab ab uvwxyz fg}
+      62  {kl uv uv uvw fg kl k}              {abcde uvw fgh uvwxy klm}
+      63  {a abc fgh u klm abcd}              {fgh pqr uv klmn fghij}
+      64  {klmn k klmn klmno pqrs pqr}        {fg kl abcde klmno uvwxy kl pq}
+      65  {uvwxyz klm fghi abc abcde kl}      {uvwxy uvw uvwxyz uvwxyz pq pqrst}
+      66  {pq klm abc pqrst fgh f}            {u abcde pqrst abcde fg}
+      67  {u pqrst kl u uvw klmno}            {u pqr pqrs fgh u p}
+      68  {abc fghi uvwxy fgh k pq}           {uv p uvwx uvwxyz ab}
+      69  {klmno f uvwxyz uvwxy klmn fg ab}   {fgh kl a pqr abcd pqr}
+      70  {fghi pqrst pqrst uv a}             {uvwxy k p uvw uvwx a}
+      71  {a fghij f p uvw}                   {klm fg abcd abcde klmno pqrs}
+      72  {uv uvwx uvwx uvw klm}              {uv fghi klmno uvwxy uvw}
+      73  {kl uvwxy ab f pq klm u}            {uvwxy klmn klm abcd pq fg k}
+      74  {uvw pqrst abcd uvwxyz ab}          {fgh fgh klmn abc pq}
+      75  {uvwxyz klm pq abcd klmno pqr uvwxyz} {kl f a fg pqr klmn}
+      76  {uvw uvwxy pqr k pqrst kl}          {uvwxy abc uvw uvw u}
+      77  {fgh klm u uvwxyz f uvwxy abcde}    {uv abcde klmno u u ab}
+      78  {klmno abc pq pqr fgh}              {p uv abcd fgh abc u k}
+      79  {fg pqr uvw pq uvwx}                {uv uvw fghij pqrs fg p}
+      80  {abcd pqrs uvwx uvwxy uvwx}         {u uvw pqrst pqr abcde pqrs kl}
+      81  {uvwxyz klm pq uvwxy fghij}         {p pq klm fghij u a a}
+      82  {uvwx k uvwxyz klmno pqrst kl}      {abcde p f pqrst abcd uvwxyz p}
+      83  {abcd abcde klm pqrst uvwxyz}       {uvw pqrst u p uvwxyz a pqrs}
+      84  {k klm abc uv uvwxy klm klmn}       {k abc pqr a abc p kl}
+      85  {klmn abcd pqrs p pq klm a}         {klmn kl ab uvw pq}
+      86  {klmn a pqrs abc uvw pqrst}         {a pqr kl klm a k f}
+      87  {pqrs ab uvwx uvwxy a pqr f}        {fg klm uvwx pqr pqr}
+      88  {klmno ab k kl u uvwxyz}            {uv kl uvw fghi uv uvw}
+      89  {pq fghi pqrst klmn uvwxy abc pqrs} {fg f f fg abc abcde klm}
+      90  {kl a k fghi uvwx fghi u}           {ab uvw pqr fg a p abc}
+      91  {uvwx pqrs klmno ab fgh uvwx}       {pqr uvwx abc kl f klmno kl}
+      92  {fghij pq pqrs fghij f pqrst}       {u abcde fg pq pqr fgh k}
+      93  {fgh u pqrs abcde klmno abc}        {abc fg pqrst pqr abcde}
+      94  {uvwx p abc f pqr p}                {k pqrs kl klm abc fghi klm}
+      95  {kl p klmno uvwxyz klmn}            {fghi ab a fghi pqrs kl}
+      96  {pqr fgh pq uvwx a}                 {uvw klm klmno fg uvwxy uvwx}
+      97  {fg abc uvwxyz fghi pqrst pq}       {abc k a ab abcde f}
+      98  {uvwxy fghi uvwxy u abcde abcde uvw} {klmn uvwx pqrs uvw uvwxy abcde}
+      99  {pq fg fghi uvwx uvwx fghij uvwxy}  {klmn klmn f abc fg a}
+    } {
+      execsql {
+        INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b);
+      }
+    }
+  } {}
+  
+  proc prefix_query {prefix} {
+    set ret [list]
+    db eval {SELECT rowid, a, b FROM t1 ORDER BY rowid DESC} {
+      if {[lsearch -glob $a $prefix]>=0 || [lsearch -glob $b $prefix]>=0} {
+        lappend ret $rowid
+      }
+    }
+    return $ret
+  }
+  
+  foreach {bAsc sql} {
+    0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix}
+    1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid ASC}
+  } {
+    foreach {tn prefix} {
+      1  {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} 
+      6  {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*}
+      11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*}
+      16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*}
+      21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*}
+      27 {x*}
+    } {
+      set res [prefix_query $prefix]
+      if {$bAsc} {
+        set res [lsort -integer -increasing $res]
+      }
+      set n [llength $res]
+      do_execsql_test $T.$bAsc.$tn.$n $sql $res
+    }
+  }
+}
+
+finish_test
+

test/fts5ae.test

@@ -0,0 +1,279 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ae
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES('hello', 'world');
+  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1 VALUES('world', 'hello');
+  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1 2}
+
+do_execsql_test 1.3 {
+  INSERT INTO t1 VALUES('world', 'world');
+  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1 2}
+
+do_execsql_test 1.4.1 {
+  INSERT INTO t1 VALUES('hello', 'hello');
+}
+
+do_execsql_test 1.4.2 {
+  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1 2 4}
+
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
+  INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
+  INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
+}
+
+do_execsql_test 2.1 {
+  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
+  WHERE t2 MATCH 'm' ORDER BY rowid;
+} {
+  1 {{0.5 1.0 1.2}} 
+  2 {{0.7 1.5}}
+}
+
+do_execsql_test 2.2 {
+  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
+  WHERE t2 MATCH 'u OR q' ORDER BY rowid;
+} {
+  1 {0.0 {}}
+  2 {{} {0.2 0.10}}
+}
+
+do_execsql_test 2.3 {
+  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
+  WHERE t2 MATCH 'y:o' ORDER BY rowid;
+} {
+  1 {{1.3 1.7}}
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(x, y);
+  INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
+  INSERT INTO t3 VALUES( 'r c', '');
+}
+
+do_execsql_test 3.1 {
+  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(a b)';
+} {
+  1 {{0.6 0.10 0.12} {0.9 0.15}}
+}
+
+do_execsql_test 3.2 {
+  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(r c)';
+} {
+  2 {0.0 0.1}
+}
+
+do_execsql_test 3.3 {
+  INSERT INTO t3 
+  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
+  SELECT rowid, fts5_test(t3, 'poslist') 
+  FROM t3 WHERE t3 MATCH 'a OR b AND c';
+} {
+  3 {0.5 {} {}} 
+  1 {{0.6 0.10 0.12} {0.9 0.15} 1.2}
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
+  INSERT INTO t4 
+  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
+}
+
+do_execsql_test 4.1 {
+  SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
+} {
+  1 {0.5 {} {}} 
+}
+
+#-------------------------------------------------------------------------
+# Test that the xColumnSize() and xColumnAvgsize() APIs work.
+#
+
+reset_db
+do_execsql_test 5.1 {
+  CREATE VIRTUAL TABLE t5 USING fts5(x, y);
+  INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
+  INSERT INTO t5 VALUES('', 'a');
+  INSERT INTO t5 VALUES('a', '');
+}
+do_execsql_test 5.2 {
+  SELECT rowid, fts5_test(t5, 'columnsize') FROM t5 WHERE t5 MATCH 'a'
+  ORDER BY rowid DESC;
+} {
+  3 {1 0}
+  2 {0 1}
+  1 {4 6}
+}
+
+do_execsql_test 5.2 {
+  SELECT rowid, fts5_test(t5, 'columntext') FROM t5 WHERE t5 MATCH 'a'
+  ORDER BY rowid DESC;
+} {
+  3 {a {}}
+  2 {{} a}
+  1 {{a b c d} {e f g h i j}}
+}
+
+do_execsql_test 5.3 {
+  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
+  ORDER BY rowid DESC;
+} {
+  3 {5 7}
+  2 {5 7}
+  1 {5 7}
+}
+
+do_execsql_test 5.4 {
+  INSERT INTO t5 VALUES('x y z', 'v w x y z');
+  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
+  ORDER BY rowid DESC;
+} {
+  3 {8 12}
+  2 {8 12}
+  1 {8 12}
+}
+
+#-------------------------------------------------------------------------
+# Test the xTokenize() API
+#
+reset_db
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
+  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
+  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
+}
+
+do_execsql_test 6.2 {
+  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
+} {
+  2 {{horatio than are} {dreamt of in your philosophy}}
+  1 {{there are more} {things in heaven and earth}}
+}
+
+#-------------------------------------------------------------------------
+# Test the xQueryPhrase() API
+#
+reset_db
+do_execsql_test 7.1 {
+  CREATE VIRTUAL TABLE t7 USING fts5(x, y);
+}
+do_test 7.2 {
+  foreach {x y} {
+    {q i b w s a a e l o} {i b z a l f p t e u}
+    {b a z t a l o x d i} {b p a d b f h d w y}
+    {z m h n p p u i e g} {v h d v b x j j c z}
+    {a g i m v a u c b i} {p k s o t l r t b m}
+    {v v c j o d a s c p} {f f v o k p o f o g}
+  } {
+    execsql {INSERT INTO t7 VALUES($x, $y)}
+  }
+  execsql { SELECT count(*) FROM t7 }
+} {5}
+
+foreach {tn q res} {
+  1 a {{4 2}}
+  2 b {{3 4}}
+  3 c {{2 1}}
+  4 d {{2 2}}
+  5 {a AND b} {{4 2} {3 4}}
+  6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
+} {
+  do_execsql_test 7.3.$tn { 
+    SELECT fts5_test(t7, 'queryphrase') FROM t7 WHERE t7 MATCH $q LIMIT 1
+  } [list $res]
+}
+
+do_execsql_test 7.4 {
+  SELECT fts5_test(t7, 'rowcount') FROM t7 WHERE t7 MATCH 'a';
+} {5 5 5 5}
+
+#do_execsql_test 7.4 {
+#  SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a';
+#} {5 5 5 5}
+#
+
+#-------------------------------------------------------------------------
+#
+do_test 8.1 {
+  execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) }
+  foreach {rowid x y} {
+     0 {A o}   {o o o C o o o o o o o o}
+     1 {o o B} {o o o C C o o o o o o o}
+     2 {A o o} {o o o o D D o o o o o o}
+     3 {o B}   {o o o o o D o o o o o o}
+     4 {E o G} {H o o o o o o o o o o o}
+     5 {F o G} {I o J o o o o o o o o o}
+     6 {E o o} {H o J o o o o o o o o o}
+     7 {o o o} {o o o o o o o o o o o o}
+     9 {o o o} {o o o o o o o o o o o o}
+  } {
+    execsql { INSERT INTO t8(rowid, x, y) VALUES($rowid, $x, $y) }
+  }
+} {}
+
+foreach {tn q res} {
+  1 {a} {0 2}
+  2 {b} {3 1}
+  3 {c} {1 0}
+  4 {d} {2 3}
+  5 {g AND (e OR f)} {5 4}
+  6 {j AND (h OR i)} {5 6}
+} {
+  do_execsql_test 8.2.$tn.1 {
+    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8) DESC;
+  } $res
+
+  do_execsql_test 8.2.$tn.2 {
+    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank DESC;
+  } $res
+
+  do_execsql_test 8.3.$tn.3 {
+    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank DESC;
+  } $res
+}
+
+
+finish_test
+

test/fts5af.test

@@ -0,0 +1,145 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+# 
+# More specifically, the tests in this file focus on the built-in 
+# snippet() function.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5af
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+}
+
+proc do_snippet_test {tn doc match res} {
+
+  uplevel #0 [list set v1 $doc]
+  uplevel #0 [list set v2 $match]
+
+  do_execsql_test $tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES($v1, NULL);
+    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
+  } [list $res]
+
+  do_execsql_test $tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES(NULL, $v1);
+    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
+  } [list $res]
+
+  do_execsql_test $tn.3 {
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES($v1, NULL);
+    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2
+    ORDER BY rank DESC;
+  } [list $res]
+
+
+}
+
+
+foreach {tn doc res} {
+
+  1.1 {X o o o o o o} {[X] o o o o o o}
+  1.2 {o X o o o o o} {o [X] o o o o o}
+  1.3 {o o X o o o o} {o o [X] o o o o}
+  1.4 {o o o X o o o} {o o o [X] o o o}
+  1.5 {o o o o X o o} {o o o o [X] o o}
+  1.6 {o o o o o X o} {o o o o o [X] o}
+  1.7 {o o o o o o X} {o o o o o o [X]}
+
+  2.1 {X o o o o o o o} {[X] o o o o o o...}
+  2.2 {o X o o o o o o} {o [X] o o o o o...}
+  2.3 {o o X o o o o o} {o o [X] o o o o...}
+  2.4 {o o o X o o o o} {o o o [X] o o o...}
+  2.5 {o o o o X o o o} {...o o o [X] o o o}
+  2.6 {o o o o o X o o} {...o o o o [X] o o}
+  2.7 {o o o o o o X o} {...o o o o o [X] o}
+  2.8 {o o o o o o o X} {...o o o o o o [X]}
+
+  3.1 {X o o o o o o o o} {[X] o o o o o o...}
+  3.2 {o X o o o o o o o} {o [X] o o o o o...}
+  3.3 {o o X o o o o o o} {o o [X] o o o o...}
+  3.4 {o o o X o o o o o} {o o o [X] o o o...}
+  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
+  3.6 {o o o o o X o o o} {...o o o [X] o o o}
+  3.7 {o o o o o o X o o} {...o o o o [X] o o}
+  3.8 {o o o o o o o X o} {...o o o o o [X] o}
+  3.9 {o o o o o o o o X} {...o o o o o o [X]}
+
+  4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
+  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
+  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
+
+  5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
+  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
+  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
+  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
+
+  6.1 {X o o o X o o o} {[X] o o o [X] o o...}
+  6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
+  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
+  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
+  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
+
+  7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
+  7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
+  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
+  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
+  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
+  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
+} {
+  do_snippet_test 1.$tn $doc X $res
+}
+
+foreach {tn doc res} {
+  1.1 {X Y o o o o o} {[X Y] o o o o o}
+  1.2 {o X Y o o o o} {o [X Y] o o o o}
+  1.3 {o o X Y o o o} {o o [X Y] o o o}
+  1.4 {o o o X Y o o} {o o o [X Y] o o}
+  1.5 {o o o o X Y o} {o o o o [X Y] o}
+  1.6 {o o o o o X Y} {o o o o o [X Y]}
+
+  2.1 {X Y o o o o o o} {[X Y] o o o o o...}
+  2.2 {o X Y o o o o o} {o [X Y] o o o o...}
+  2.3 {o o X Y o o o o} {o o [X Y] o o o...}
+  2.4 {o o o X Y o o o} {...o o [X Y] o o o}
+  2.5 {o o o o X Y o o} {...o o o [X Y] o o}
+  2.6 {o o o o o X Y o} {...o o o o [X Y] o}
+  2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+
+  3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
+  3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
+  3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
+  3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
+  3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
+  3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
+  3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
+  3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+
+} {
+  do_snippet_test 1.$tn $doc "X + Y" $res
+}
+
+finish_test
+

test/fts5ag.test

@@ -0,0 +1,137 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ag
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# This file attempts to verify that the extension APIs work with 
+# "ORDER BY rank" queries. This is done by comparing the results of
+# the fts5_test() function when run with queries of the form:
+#
+#      ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC]
+#
+# and
+#
+#      ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC]
+#
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, z);
+}
+
+do_test 1.1 {
+  foreach {x y z} {
+    {j s m y m r n l u k} {z k f u z g h s w g} {r n o s s b v n w w}
+    {m v g n d x q r r s} {q t d a q a v l h j} {s k l f s i n v q v}
+    {m f f d h h s o h a} {y e v r q i u m h d} {b c k q m z l z h n}
+    {j e m v k p e c j m} {m p v z d x l n i a} {v p u p m t p q i f}
+    {v r w l e e t d z p} {c s b w k m n k o u} {w g y f v w v w v p}
+    {k d g o u j p z n o} {t g e q l z i g b j} {f i q q j y h b g h}
+    {j s w x o t j b t m} {v a v v r t x c q a} {r t k x w u l h a g}
+    {j y b i u d e m d w} {y s o j h i n a u p} {n a g b u c w e b m}
+    {b c k s c w j p w b} {m o c o w o b d q q} {n t y o y z y r z e}
+    {p n q l e l h z q c} {n s e i h c v b b u} {m p d i t a o o f f}
+    {k c o n v e z l b m} {s m n i n s d e s u} {t a u e q d a o u c}
+    {h d t o i a g b b p} {k x c i g f g b b k} {x f i v n a n n j i}
+    {f z k r b u s k z e} {n z v z w l e r h t} {t i s v v a v p n s}
+    {k f e c t z r e f d} {f m g r c w q k b v} {v y s y f r b f e f}
+    {z r c t d q q h x b} {u c g z n z u v s s} {y t n f f x b f d x}
+    {u n p n u t i m e j} {p j j d m f k p m z} {d o l v c o e a h w}
+    {h o q w t f v i c y} {c q u n r z s l l q} {z x a q w s b w s y}
+    {y m s x k i m n x c} {b i a n v h z n k a} {w l q p b h h g d y}
+    {z v s j f p v l f w} {c s b i z e k i g c} {x b v d w j f e d z}
+    {r k k j e o m k g b} {h b d c h m y b t u} {u j s h k z c u d y}
+    {v h i v s y z i k l} {d t m w q w c a z p} {r s e s x v d w k b}
+    {u r e q j y h o o s} {x x z r x y t f j s} {k n h x i i u e c v}
+    {q l f d a p w l q o} {y z q w j o p b o v} {s u h z h f d f n l}
+    {q o e o x x l g q i} {j g m h q q w c d b} {o m d h w a g b f n}
+    {m x k t s s y l v a} {j x t c a u w b w g} {n f j b v x y p u t}
+    {u w k a q b u w k w} {a h j u o w f s k p} {j o f s h y t j h g}
+    {x v b l m t l m h l} {t p y i y i q b q a} {k o o z w a c h c f}
+    {j g c d k w b d t v} {a k v c m a v h v p} {i c a i j g h l j h}
+    {l m v l c z j b p b} {z p z f l n k i b a} {j v q k g i x g i b}
+    {m c i w u z m i s z} {i z r f n l q z k w} {x n b p b q r g i z}
+    {d g i o o x l f x d} {r t m f b n q y c b} {i u g k w x n m p o}
+    {t o s i q d z x d t} {v a k s q z j c o o} {z f n n r l y w v v}
+    {w k h d t l j g n n} {r z m v y b l n c u} {v b v s c l n k g v}
+    {m a g r a b u u n z} {u y l h v w v k b f} {x l p g i s j f x v}
+    {v s g x k z a k a r} {l t g v j q l k p l} {f h n a x t v s t y}
+    {z u v u x p s j y t} {g b q e e g l n w g} {e n p j i g j f u r}
+    {q z l t w o l m p e} {t s g h r p r o t z} {y b f a o n u m z g}
+    {d t w n y b o g f o} {d a j e r l g g s h} {d z e l w q l t h f}
+    {f l u w q v x j a h} {f n u l l d m h h w} {d x c c e r o d q j}
+    {b y f q s q f u l g} {u z w l f d b i a g} {m v q b g u o z e z}
+    {h z p t s e x i v m} {l h q m e o x x x j} {e e d n p r m g j f}
+    {k h s g o n s d a x} {u d t t s j o v h a} {z r b a e u v o e s}
+    {m b b g a f c p a t} {w c m j o d b l g e} {f p j p m o s y v j}
+    {c r n h d w c a b l} {s g e u s d n j b g} {b o n a x a b x y l}
+    {r h u x f c d z n o} {x y l g u m i i w d} {t f h b z v r s r g}
+    {t i o r b v g g p a} {d x l u q k m o s u} {j f h t u n z u k m}
+    {g j t y d c n j y g} {w e s k v c w i g t} {g a h r g v g h r o}
+    {e j l a q j g i n h} {d z k c u p n u p p} {t u e e v z v r r g}
+    {l j s g k j k h z l} {p v d a t x d e q u} {r l u z b m g k s j}
+    {i e y d u x d i n l} {p f z k m m w p u l} {z l p m r q w n d a}
+  } {
+    execsql { INSERT INTO t1 VALUES($x, $y, $z) }
+  }
+  set {} {}
+} {}
+
+proc do_fts5ag_test {tn E} {
+  set q1 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
+  set q2 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}
+
+  set res [execsql $q1]
+  set expected [execsql $q2]
+  uplevel [list do_test $tn.1 [list set {} $res] $expected]
+
+  append q1 " DESC"
+  append q2 " DESC"
+
+  set res [execsql $q1]
+  set expected [execsql $q2]
+  uplevel [list do_test $tn.2 [list set {} $res] $expected]
+}
+
+foreach {tn expr} {
+  2.1 a
+  2.2 b
+  2.3 c
+  2.4 d
+
+  2.5 {"m m"}
+  2.6 {e + s}
+
+  3.0 {a AND b}
+  3.1 {a OR b}
+  3.2 {b OR c AND d}
+  3.3 {NEAR(c d)}
+} {
+  do_fts5ag_test $tn $expr
+
+  if {[set_test_counter errors]} break
+}
+
+
+
+finish_test
+

test/fts5ea.test

@@ -0,0 +1,84 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ea
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+proc do_syntax_error_test {tn expr err} {
+  set ::se_expr $expr
+  do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err]
+}
+
+proc do_syntax_test {tn expr res} {
+  set ::se_expr $expr
+  do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
+}
+
+foreach {tn expr res} {
+  1  {abc}                           {"abc"}
+  2  {abc def}                       {"abc" AND "def"}
+  3  {abc*}                          {"abc" *}
+  4  {"abc def ghi" *}               {"abc" + "def" + "ghi" *}
+  5  {one AND two}                   {"one" AND "two"}
+  6  {one+two}                       {"one" + "two"}
+  7  {one AND two OR three}          {("one" AND "two") OR "three"}
+  8  {one OR two AND three}          {"one" OR ("two" AND "three")}
+  9  {NEAR(one two)}                 {NEAR("one" "two", 10)}
+  10 {NEAR("one three"* two, 5)}     {NEAR("one" + "three" * "two", 5)}
+} {
+  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
+}
+
+foreach {tn expr res} {
+  1 {c1:abc}                           
+    {c1 : "abc"}
+  2 {c2 : NEAR(one two) c1:"hello world"} 
+    {c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"}
+} {
+  do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res]
+}
+
+breakpoint
+foreach {tn expr err} {
+  1 {AND}                          {fts5: syntax error near "AND"}
+  2 {abc def AND}                  {fts5: syntax error near ""}
+  3 {abc OR AND}                   {fts5: syntax error near "AND"}
+  4 {(a OR b) abc}                 {fts5: syntax error near "abc"}
+  5 {NEaR (a b)}                   {fts5: syntax error near "NEaR"}
+  6 {(a OR b) NOT c)}              {fts5: syntax error near ")"}
+  7 {nosuch: a nosuch2: b}         {no such column: nosuch}
+  8 {addr: a nosuch2: b}           {no such column: nosuch2}
+} {
+  do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
+}
+
+
+
+# do_syntax_error_test 1.0 {NOT} {syntax error near "NOT"}
+
+
+
+# do_catchsql_test 1.1 { 
+ #  SELECT fts5_expr('a OR b NOT c') 
+#} {0 {"a" OR "b" NOT "c"}}
+
+
+#do_execsql_test 1.0 { SELECT fts5_expr('a') } {{"a"}}
+
+finish_test

test/permutations.test

@@ -222,6 +222,13 @@ test_suite "fts3" -prefix "" -description {
   fts4growth.test fts4growth2.test
 }
 
+test_suite "fts5" -prefix "" -description {
+  All FTS5 tests.
+} -files {
+  fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test fts5ea.test
+  fts5af.test fts5ag.test
+}
+
 test_suite "nofaultsim" -prefix "" -description {
   "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
   This test suite is the same as the "quick" tests, except that some files

tool/loadfts.c

@@ -0,0 +1,204 @@
+/*
+** 2013-06-10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <assert.h>
+#include <string.h>
+#include <errno.h>
+#include <dirent.h>
+#include "sqlite3.h"
+
+/*
+** Implementation of the "readtext(X)" SQL function.  The entire content
+** of the file named X is read and returned as a TEXT value. It is assumed
+** the file contains UTF-8 text. NULL is returned if the file does not 
+** exist or is unreadable.
+*/
+static void readfileFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const char *zName;
+  FILE *in;
+  long nIn;
+  void *pBuf;
+
+  zName = (const char*)sqlite3_value_text(argv[0]);
+  if( zName==0 ) return;
+  in = fopen(zName, "rb");
+  if( in==0 ) return;
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc( nIn );
+  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
+    sqlite3_result_text(context, pBuf, nIn, sqlite3_free);
+  }else{
+    sqlite3_free(pBuf);
+  }
+  fclose(in);
+}
+
+/*
+** Print usage text for this program and exit.
+*/
+static void showHelp(const char *zArgv0){
+  printf("\n"
+"Usage: %s SWITCHES... DB\n"
+"\n"
+"  This program opens the database named on the command line and attempts to\n"
+"  create an FTS table named \"fts\" with a single column. If successful, it\n"
+"  recursively traverses the directory named by the -dir option and inserts\n"
+"  the contents of each file into the fts table. All files are assumed to\n"
+"  contain UTF-8 text.\n"
+"\n"
+"Switches are:\n"
+"  -fts [345]       FTS version to use (default=5)\n"
+"  -idx [01]        Create a mapping from filename to rowid (default=0)\n"
+"  -dir <path>      Root of directory tree to load data from (default=.)\n"
+, zArgv0
+);
+  exit(1);
+}
+
+/*
+** Exit with a message based on the argument and the current value of errno.
+*/
+static void error_out(const char *zText){
+  fprintf(stderr, "%s: %s\n", zText, strerror(errno));
+  exit(-1);
+}
+
+/*
+** Exit with a message based on the first argument and the error message
+** currently stored in database handle db.
+*/
+static void sqlite_error_out(const char *zText, sqlite3 *db){
+  fprintf(stderr, "%s: %s\n", zText, sqlite3_errmsg(db));
+  exit(-1);
+}
+
+/*
+** Context object for visit_file().
+*/
+typedef struct VisitContext VisitContext;
+struct VisitContext {
+  sqlite3 *db;                    /* Database handle */
+  sqlite3_stmt *pInsert;          /* INSERT INTO fts VALUES(readtext(:1)) */
+};
+
+/*
+** Callback used with traverse(). The first argument points to an object
+** of type VisitContext. This function inserts the contents of the text
+** file zPath into the FTS table.
+*/
+void visit_file(void *pCtx, const char *zPath){
+  int rc;
+  VisitContext *p = (VisitContext*)pCtx;
+  /* printf("%s\n", zPath); */
+  sqlite3_bind_text(p->pInsert, 1, zPath, -1, SQLITE_STATIC);
+  sqlite3_step(p->pInsert);
+  rc = sqlite3_reset(p->pInsert);
+  if( rc!=SQLITE_OK ) sqlite_error_out("insert", p->db);
+}
+
+/*
+** Recursively traverse directory zDir. For each file that is not a 
+** directory, invoke the supplied callback with its path.
+*/
+static void traverse(
+  const char *zDir,               /* Directory to traverse */
+  void *pCtx,                     /* First argument passed to callback */
+  void (*xCallback)(void*, const char *zPath)
+){
+  DIR *d;
+  struct dirent *e;
+
+  d = opendir(zDir);
+  if( d==0 ) error_out("opendir()");
+
+  for(e=readdir(d); e; e=readdir(d)){
+    if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue;
+    char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name);
+    if (e->d_type & DT_DIR) {
+      traverse(zPath, pCtx, xCallback);
+    }else{
+      xCallback(pCtx, zPath);
+    }
+    sqlite3_free(zPath);
+  }
+
+  closedir(d);
+}
+
+int main(int argc, char **argv){
+  int iFts = 5;                   /* Value of -fts option */
+  int bMap = 0;                   /* True to create mapping table */
+  const char *zDir = ".";         /* Directory to scan */
+  int i;
+  int rc;
+  sqlite3 *db;
+  char *zSql;
+  VisitContext sCtx;
+
+  if( argc % 2 ) showHelp(argv[0]);
+
+  for(i=1; i<(argc-1); i+=2){
+    char *zOpt = argv[i];
+    char *zArg = argv[i+1];
+    if( strcmp(zOpt, "-fts")==0 ){
+      iFts = atoi(zArg);
+      if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]);
+    }
+    else if( strcmp(zOpt, "-idx")==0 ){
+      bMap = atoi(zArg);
+      if( bMap!=0 && bMap!=1 ) showHelp(argv[0]);
+    }
+    else if( strcmp(zOpt, "-dir")==0 ){
+      zDir = zArg;
+    }
+  }
+
+  /* Open the database file */
+  rc = sqlite3_open(argv[argc-1], &db);
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_open()", db);
+
+  rc = sqlite3_create_function(db, "readtext", 1, SQLITE_UTF8, 0,
+                               readfileFunc, 0, 0);
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_create_function()", db);
+
+  /* Create the FTS table */
+  zSql = sqlite3_mprintf("CREATE VIRTUAL TABLE fts USING fts%d(content)", iFts);
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db);
+  sqlite3_free(zSql);
+
+  /* Compile the INSERT statement to write data to the FTS table. */
+  memset(&sCtx, 0, sizeof(VisitContext));
+  sCtx.db = db;
+  rc = sqlite3_prepare_v2(db, 
+      "INSERT INTO fts VALUES(readtext(?))", -1, &sCtx.pInsert, 0
+  );
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_prepare_v2(1)", db);
+
+  /* Load all files in the directory hierarchy into the FTS table. */
+  traverse(zDir, (void*)&sCtx, visit_file);
+
+  /* Clean up and exit. */
+  sqlite3_finalize(sCtx.pInsert);
+  sqlite3_close(db);
+  return 0;
+}

tool/mksqlite3c.tcl

@@ -97,6 +97,8 @@ foreach hdr {
    fts3Int.h
    fts3_hash.h
    fts3_tokenizer.h
+   fts5.h
+   fts5Int.h
    hash.h
    hwtime.h
    keywordhash.h
@@ -328,6 +330,15 @@ foreach file {
    fts3_unicode.c
    fts3_unicode2.c
 
+   fts5_aux.c
+   fts5_buffer.c
+   fts5.c
+   fts5_config.c
+   fts5_expr.c
+   fts5_index.c
+   fts5parse.c
+   fts5_storage.c
+
    rtree.c
    icu.c
    fts3_icu.c