paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a snippet() function to fts5.

Browse Source 
FossilOrigin-Name: bdc58fd28a63ac9632c3df6c7768a9a236566605
This commit is contained in:
dan
2014-07-23 19:31:56 +00:00
parent a5983da6e1
commit b30860abe2
10 changed files with 626 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5.c
+2 -3
View File
@@ -686,13 +686,12 @@ static int fts5ApiPoslist(
Fts5Context *pCtx,
int iPhrase,
int *pi,
int *piCol,
int *piOff
i64 *piPos
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
const u8 *a; int n; /* Poslist for phrase iPhrase */
n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);
return sqlite3Fts5PoslistNext(a, n, pi, piCol, piOff);
return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}
static void fts5ApiCallback(
ext/fts5/fts5.h
+6 -1
View File
@@ -69,6 +69,8 @@ typedef void (*fts5_extension_function)(
** xPoslist:
** Iterate through instances of phrase iPhrase in the current row.
**
** At EOF, a non-zero value is returned and output variable iPos set to -1.
**
** xTokenize:
** Tokenize text using the tokenizer belonging to the FTS5 table.
*/
@@ -91,9 +93,12 @@ struct Fts5ExtensionApi {
sqlite3_int64 (*xRowid)(Fts5Context*);
int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, int *piCol, int *piOff);
int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, sqlite3_int64 *piPos);
};
#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
/*
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/
ext/fts5/fts5_aux.c
+405 -3
View File
@@ -13,12 +13,413 @@
#include "fts5Int.h"
typedef struct SnippetPhrase SnippetPhrase;
typedef struct SnippetIter SnippetIter;
typedef struct SnippetCtx SnippetCtx;
struct SnippetPhrase {
u64 mask; /* Current mask */
int nToken; /* Tokens in this phrase */
int i; /* Current offset in phrase poslist */
i64 iPos; /* Next position in phrase (-ve -> EOF) */
};
struct SnippetIter {
i64 iLast; /* Last token position of current snippet */
int nScore; /* Score of current snippet */
const Fts5ExtensionApi *pApi;
Fts5Context *pFts;
u64 szmask; /* Mask used to on SnippetPhrase.mask */
int nPhrase; /* Number of phrases */
SnippetPhrase aPhrase[0]; /* Array of size nPhrase */
};
struct SnippetCtx {
int iFirst; /* Offset of first token to record */
int nToken; /* Size of aiStart[] and aiEnd[] arrays */
int iSeen; /* Set to largest offset seen */
int *aiStart;
int *aiEnd;
};
static int fts5SnippetCallback(
void *pContext, /* Pointer to Fts5Buffer object */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd, /* End offset of token */
int iPos /* Position offset of token */
){
int rc = SQLITE_OK;
SnippetCtx *pCtx = (SnippetCtx*)pContext;
int iOff = iPos - pCtx->iFirst;
if( iOff>=0 ){
if( iOff < pCtx->nToken ){
pCtx->aiStart[iOff] = iStart;
pCtx->aiEnd[iOff] = iEnd;
}
pCtx->iSeen = iPos;
if( iOff>=pCtx->nToken ) rc = SQLITE_DONE;
}
return rc;
}
/*
** Set pIter->nScore to the score for the current entry.
*/
static void fts5SnippetCalculateScore(SnippetIter *pIter){
int i;
int nScore = 0;
assert( pIter->iLast>=0 );
for(i=0; i<pIter->nPhrase; i++){
SnippetPhrase *p = &pIter->aPhrase[i];
u64 mask = p->mask;
if( mask ){
u64 j;
nScore += 1000;
for(j=1; j & pIter->szmask; j<<=1){
if( mask & j ) nScore++;
}
}
}
pIter->nScore = nScore;
}
/*
** Allocate a new snippet iter.
*/
static int fts5SnippetIterNew(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
int nToken, /* Number of tokens in snippets */
SnippetIter **ppIter /* OUT: New object */
){
int i; /* Counter variable */
SnippetIter *pIter; /* New iterator object */
int nByte; /* Bytes of space to allocate */
int nPhrase; /* Number of phrases in query */
*ppIter = 0;
nPhrase = pApi->xPhraseCount(pFts);
nByte = sizeof(SnippetIter) + nPhrase * sizeof(SnippetPhrase);
pIter = (SnippetIter*)sqlite3_malloc(nByte);
if( pIter==0 ) return SQLITE_NOMEM;
memset(pIter, 0, nByte);
pIter->nPhrase = nPhrase;
pIter->pApi = pApi;
pIter->pFts = pFts;
pIter->szmask = ((u64)1 << nToken) - 1;
assert( nToken<=63 );
for(i=0; i<nPhrase; i++){
pIter->aPhrase[i].nToken = pApi->xPhraseSize(pFts, i);
}
*ppIter = pIter;
return SQLITE_OK;
}
/*
** Set the iterator to point to the first candidate snippet.
*/
static void fts5SnippetIterFirst(SnippetIter *pIter){
const Fts5ExtensionApi *pApi = pIter->pApi;
Fts5Context *pFts = pIter->pFts;
int i; /* Used to iterate through phrases */
SnippetPhrase *pMin = 0; /* Phrase with first match */
memset(pIter->aPhrase, 0, sizeof(SnippetPhrase) * pIter->nPhrase);
for(i=0; i<pIter->nPhrase; i++){
SnippetPhrase *p = &pIter->aPhrase[i];
p->nToken = pApi->xPhraseSize(pFts, i);
pApi->xPoslist(pFts, i, &p->i, &p->iPos);
if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ){
pMin = p;
}
}
assert( pMin );
pIter->iLast = pMin->iPos + pMin->nToken - 1;
pMin->mask = 0x01;
pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
fts5SnippetCalculateScore(pIter);
}
/*
** Advance the snippet iterator to the next candidate snippet.
*/
static void fts5SnippetIterNext(SnippetIter *pIter){
const Fts5ExtensionApi *pApi = pIter->pApi;
Fts5Context *pFts = pIter->pFts;
int nPhrase = pIter->nPhrase;
int i; /* Used to iterate through phrases */
SnippetPhrase *pMin = 0;
for(i=0; i<nPhrase; i++){
SnippetPhrase *p = &pIter->aPhrase[i];
if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ) pMin = p;
}
if( pMin==0 ){
/* pMin==0 indicates that the SnippetIter is at EOF. */
pIter->iLast = -1;
}else{
i64 nShift = pMin->iPos - pIter->iLast;
assert( nShift>=0 );
for(i=0; i<nPhrase; i++){
SnippetPhrase *p = &pIter->aPhrase[i];
if( nShift>=63 ){
p->mask = 0;
}else{
p->mask = p->mask << (int)nShift;
p->mask &= pIter->szmask;
}
}
pIter->iLast = pMin->iPos;
pMin->mask |= 0x01;
fts5SnippetCalculateScore(pIter);
pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
}
}
static void fts5SnippetIterFree(SnippetIter *pIter){
if( pIter ){
sqlite3_free(pIter);
}
}
static int fts5SnippetText(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
SnippetIter *pIter, /* Snippet to write to buffer */
int nToken, /* Size of desired snippet in tokens */
const char *zStart,
const char *zFinal,
const char *zEllip,
Fts5Buffer *pBuf /* Write output to this buffer */
){
SnippetCtx ctx;
int i;
u64 all = 0;
const char *zCol; /* Column text to extract snippet from */
int nCol; /* Size of column text in bytes */
int rc;
int nShift;
rc = pApi->xColumnText(pFts, FTS5_POS2COLUMN(pIter->iLast), &zCol, &nCol);
if( rc!=SQLITE_OK ) return rc;
/* At this point pIter->iLast is the offset of the last token in the
** proposed snippet. However, in all cases pIter->iLast contains the
** final token of one of the phrases. This makes the snippet look
** unbalanced. For example:
**
** "...x x x x x <b>term</b>..."
**
** It is better to increase iLast a little so that the snippet looks
** more like:
**
** "...x x x <b>term</b> y y..."
**
** The problem is that there is no easy way to discover whether or not
** how many tokens are present in the column following "term".
*/
/* Set variable nShift to the number of tokens by which the snippet
** should be shifted, assuming there are sufficient tokens to the right
** of iLast in the column value. */
for(i=0; i<pIter->nPhrase; i++){
int iToken;
for(iToken=0; iToken<pIter->aPhrase[i].nToken; iToken++){
all |= (pIter->aPhrase[i].mask << iToken);
}
}
for(i=nToken-1; i>=0; i--){
if( all & ((u64)1 << i) ) break;
}
assert( i>=0 );
nShift = (nToken - i) / 2;
memset(&ctx, 0, sizeof(SnippetCtx));
ctx.nToken = nToken + nShift;
ctx.iFirst = FTS5_POS2OFFSET(pIter->iLast) - nToken + 1;
if( ctx.iFirst<0 ){
nShift += ctx.iFirst;
if( nShift<0 ) nShift = 0;
ctx.iFirst = 0;
}
ctx.aiStart = (int*)sqlite3_malloc(sizeof(int) * ctx.nToken * 2);
if( ctx.aiStart==0 ) return SQLITE_NOMEM;
ctx.aiEnd = &ctx.aiStart[ctx.nToken];
rc = pApi->xTokenize(pFts, zCol, nCol, (void*)&ctx, fts5SnippetCallback);
if( rc==SQLITE_OK ){
int i1; /* First token from input to include */
int i2; /* Last token from input to include */
int iPrint;
int iMatchto;
int iBit0;
int iLast;
int *aiStart = ctx.aiStart - ctx.iFirst;
int *aiEnd = ctx.aiEnd - ctx.iFirst;
/* Ideally we want to start the snippet with token (ctx.iFirst + nShift).
** However, this is only possible if there are sufficient tokens within
** the column. This block sets variables i1 and i2 to the first and last
** input tokens to include in the snippet. */
if( (ctx.iFirst + nShift + nToken)<=ctx.iSeen ){
i1 = ctx.iFirst + nShift;
i2 = i1 + nToken - 1;
}else{
i2 = ctx.iSeen;
i1 = ctx.iSeen - nToken + 1;
assert( i1>=0 || ctx.iFirst==0 );
if( i1<0 ) i1 = 0;
}
/* If required, append the preceding ellipsis. */
if( i1>0 ) sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%s", zEllip);
iLast = FTS5_POS2OFFSET(pIter->iLast);
iPrint = i1;
iMatchto = -1;
for(i=i1; i<=i2; i++){
/* Check if this is the first token of any phrase match. */
int ip;
for(ip=0; ip<pIter->nPhrase; ip++){
SnippetPhrase *pPhrase = &pIter->aPhrase[ip];
u64 m = (1 << (iLast - i - pPhrase->nToken + 1));
if( i<=iLast && (pPhrase->mask & m) ){
if( iMatchto<0 ){
sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s%s",
aiStart[i] - aiStart[iPrint],
&zCol[aiStart[iPrint]],
zStart
);
iPrint = i;
}
if( i>iMatchto ) iMatchto = i + pPhrase->nToken - 1;
}
}
if( i==iMatchto ){
sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s%s",
aiEnd[i] - aiStart[iPrint],
&zCol[aiStart[iPrint]],
zFinal
);
iMatchto = -1;
iPrint = i+1;
if( i<i2 ){
sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s",
aiStart[i+1] - aiEnd[i],
&zCol[aiEnd[i]]
);
}
}
}
if( iPrint<=i2 ){
sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s",
aiEnd[i2] - aiStart[iPrint],
&zCol[aiStart[iPrint]]
);
if( iMatchto>=0 ){
sqlite3Fts5BufferAppendString(&rc, pBuf, zFinal);
}
}
/* If required, append the trailing ellipsis. */
if( i2<ctx.iSeen ) sqlite3Fts5BufferAppendString(&rc, pBuf, zEllip);
}
sqlite3_free(ctx.aiStart);
return rc;
}
/*
** A default snippet() implementation. This is compatible with the FTS3
** snippet() function.
*/
static void fts5SnippetFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
const char *zStart = "<b>";
const char *zFinal = "</b>";
const char *zEllip = "<b>...</b>";
int nToken = -15;
int nAbs;
int nFrag; /* Number of fragments to return */
int rc;
SnippetIter *pIter = 0;
if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
if( nVal>=3 ) zEllip = (const char*)sqlite3_value_text(apVal[2]);
if( nVal>=4 ){
nToken = sqlite3_value_int(apVal[3]);
if( nToken==0 ) nToken = -15;
}
nAbs = nToken * (nToken<0 ? -1 : 1);
rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter);
if( rc==SQLITE_OK ){
Fts5Buffer buf; /* Result buffer */
int nBestScore = 0; /* Score of best snippet found */
int n; /* Size of column snippet is from in bytes */
int i; /* Used to iterate through phrases */
for(fts5SnippetIterFirst(pIter);
pIter->iLast>=0;
fts5SnippetIterNext(pIter)
){
if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
}
for(fts5SnippetIterFirst(pIter);
pIter->iLast>=0;
fts5SnippetIterNext(pIter)
){
if( pIter->nScore==nBestScore ) break;
}
memset(&buf, 0, sizeof(Fts5Buffer));
rc = fts5SnippetText(pApi, pFts, pIter, nAbs, zStart, zFinal, zEllip, &buf);
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)buf.p, buf.n, SQLITE_TRANSIENT);
}
sqlite3_free(buf.p);
}
fts5SnippetIterFree(pIter);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}
}
static void fts5Bm25Function(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
assert( 0 );
}
@@ -146,12 +547,13 @@ static void fts5TestFunction(
for(i=0; i<nPhrase; i++){
Fts5Buffer s2; /* List of positions for phrase/column */
int j = 0;
int iOff = 0;
int iCol = 0;
i64 iPos = 0;
int nElem = 0;
memset(&s2, 0, sizeof(s2));
while( 0==pApi->xPoslist(pFts, i, &j, &iCol, &iOff) ){
while( 0==pApi->xPoslist(pFts, i, &j, &iPos) ){
int iOff = FTS5_POS2OFFSET(iPos);
int iCol = FTS5_POS2COLUMN(iPos);
if( nElem!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s2, " ");
sqlite3Fts5BufferAppendPrintf(&rc, &s2, "%d.%d", iCol, iOff);
nElem++;
ext/fts5/fts5_buffer.c
+1
View File
@@ -146,6 +146,7 @@ int sqlite3Fts5PoslistNext64(
int i = *pi;
if( i>=n ){
/* EOF */
*piOff = -1;
return 1;
}else{
i64 iOff = *piOff;
ext/fts5/fts5_expr.c
-1
View File
@@ -389,7 +389,6 @@ static int fts5ExprNearIsMatch(Fts5ExprNearset *pNear, int *pbMatch){
int i;
int rc = SQLITE_OK;
int bMatch;
i64 iMax;
assert( pNear->nPhrase>1 );
ext/fts5/fts5_storage.c
+59
View File
@@ -486,6 +486,29 @@ int sqlite3Fts5StorageInsert(
return rc;
}
static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
Fts5Config *pConfig = p->pConfig;
char *zSql;
int rc;
zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'",
pConfig->zDb, pConfig->zName, zSuffix
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
sqlite3_stmt *pCnt = 0;
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCnt) ){
*pnRow = sqlite3_column_int64(pCnt, 0);
}
rc = sqlite3_finalize(pCnt);
}
sqlite3_free(zSql);
return rc;
}
/*
** Context object used by sqlite3Fts5StorageIntegrity().
*/
@@ -493,6 +516,7 @@ typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
struct Fts5IntegrityCtx {
i64 iRowid;
int iCol;
int szCol;
u64 cksum;
Fts5Config *pConfig;
};
@@ -512,6 +536,7 @@ static int fts5StorageIntegrityCallback(
pCtx->cksum ^= sqlite3Fts5IndexCksum(
pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken
);
pCtx->szCol = iPos+1;
return SQLITE_OK;
}
@@ -524,11 +549,17 @@ static int fts5StorageIntegrityCallback(
int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
Fts5Config *pConfig = p->pConfig;
int rc; /* Return code */
int *aColSize; /* Array of size pConfig->nCol */
i64 *aTotalSize; /* Array of size pConfig->nCol */
Fts5IntegrityCtx ctx;
sqlite3_stmt *pScan;
memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
ctx.pConfig = p->pConfig;
aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
if( !aTotalSize ) return SQLITE_NOMEM;
aColSize = (int*)&aTotalSize[pConfig->nCol];
memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
/* Generate the expected index checksum based on the contents of the
** %_content table. This block stores the checksum in ctx.cksum. */
@@ -538,6 +569,8 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
while( SQLITE_ROW==sqlite3_step(pScan) ){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
ctx.szCol = 0;
rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
ctx.iCol = i;
rc = sqlite3Fts5Tokenize(
@@ -547,12 +580,37 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
(void*)&ctx,
fts5StorageIntegrityCallback
);
if( ctx.szCol!=aColSize[i] ) rc = SQLITE_CORRUPT_VTAB;
aTotalSize[i] += ctx.szCol;
}
if( rc!=SQLITE_OK ) break;
}
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
}
/* Test that the "totals" (sometimes called "averages") record looks Ok */
if( rc==SQLITE_OK ){
int i;
rc = fts5StorageLoadTotals(p);
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( p->aTotalSize[i]!=aTotalSize[i] ) rc = SQLITE_CORRUPT_VTAB;
}
}
/* Check that the %_docsize and %_content tables contain the expected
** number of rows. */
if( rc==SQLITE_OK ){
i64 nRow;
rc = fts5StorageCount(p, "content", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;
}
if( rc==SQLITE_OK ){
i64 nRow;
rc = fts5StorageCount(p, "docsize", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;
}
/* Pass the expected checksum down to the FTS index module. It will
** verify, amongst other things, that it matches the checksum generated by
** inspecting the index itself. */
@@ -560,6 +618,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum);
}
sqlite3_free(aTotalSize);
return rc;
}
manifest
+13 -12
View File
@@ -1,5 +1,5 @@
C Fix\sDELETE\sand\sUPDATE\soperations\son\sfts5\stables.
D 2014-07-21T15:45:26.584
C Add\sa\ssnippet()\sfunction\sto\sfts5.
D 2014-07-23T19:31:56.454
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,15 +103,15 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
F ext/fts5/fts5.c 35124fe8a49868808604c6a5264bf4f23587ac99
F ext/fts5/fts5.h c77b6a4a56d80f70fc4f0444030c88724397ed10
F ext/fts5/fts5.c 6f859d444eb8be46cb3f7aba3aaae369c5b26809
F ext/fts5/fts5.h 57325b418b26dcd60be5bc8aab05b33767d81590
F ext/fts5/fts5Int.h 12d03496152b716e63a5380e396b776fbefa2065
F ext/fts5/fts5_aux.c 14961135231dd50e6c17894e649c3bbc8c042829
F ext/fts5/fts5_buffer.c 00361d4a70040ebd2c32bc349ab708ff613a1749
F ext/fts5/fts5_aux.c cba929fb13931c9b8be7d572991e648b98f14cf2
F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
F ext/fts5/fts5_expr.c 2911813db174afa28b69ccc7031b6dd80293b241
F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997
F ext/fts5/fts5_storage.c 7bb34138d134841cbe0a809467070d07013d8d7d
F ext/fts5/fts5_storage.c 9a2744f492413395a0e75f20c19b797c801a7308
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
@@ -600,6 +600,7 @@ F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
F test/fts5ae.test fe9db78201bbb87c6f82b72a14b946d0f7fc3026
F test/fts5af.test a2980528a04b67ac4690e6c02ebe9455f45c9a35
F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -767,7 +768,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
F test/permutations.test c3eb62a88337d9a5046c509dd90ba6d43debc76d
F test/permutations.test e6e951c816199693676d8c3d22d6bf54bcd719fc
F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@@ -1195,7 +1196,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 8c6b0aff3443fae4b7f0b9adcbf1514992b70653
R ceadf07a7e21667d721f7d8d3cf93a59
P d44d3a8518ff7a1a3e2c0ab97493aa590676ee8c
R 638d6826a594d773b5778bd6943c3d96
U dan
Z 2d0ad058324af420450f942c4237ae4f
Z 30db824dafb73f9c4c6895383aa25ed9
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
d44d3a8518ff7a1a3e2c0ab97493aa590676ee8c
bdc58fd28a63ac9632c3df6c7768a9a236566605
test/fts5af.test
+138
View File
@@ -0,0 +1,138 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# More specifically, the tests in this file focus on the built-in
# snippet() function.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5af
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
}
foreach {tn doc res} {
1.1 {X o o o o o o} {[X] o o o o o o}
1.2 {o X o o o o o} {o [X] o o o o o}
1.3 {o o X o o o o} {o o [X] o o o o}
1.4 {o o o X o o o} {o o o [X] o o o}
1.5 {o o o o X o o} {o o o o [X] o o}
1.6 {o o o o o X o} {o o o o o [X] o}
1.7 {o o o o o o X} {o o o o o o [X]}
2.1 {X o o o o o o o} {[X] o o o o o o...}
2.2 {o X o o o o o o} {o [X] o o o o o...}
2.3 {o o X o o o o o} {o o [X] o o o o...}
2.4 {o o o X o o o o} {o o o [X] o o o...}
2.5 {o o o o X o o o} {...o o o [X] o o o}
2.6 {o o o o o X o o} {...o o o o [X] o o}
2.7 {o o o o o o X o} {...o o o o o [X] o}
2.8 {o o o o o o o X} {...o o o o o o [X]}
3.1 {X o o o o o o o o} {[X] o o o o o o...}
3.2 {o X o o o o o o o} {o [X] o o o o o...}
3.3 {o o X o o o o o o} {o o [X] o o o o...}
3.4 {o o o X o o o o o} {o o o [X] o o o...}
3.5 {o o o o X o o o o} {...o o o [X] o o o...}
3.6 {o o o o o X o o o} {...o o o [X] o o o}
3.7 {o o o o o o X o o} {...o o o o [X] o o}
3.8 {o o o o o o o X o} {...o o o o o [X] o}
3.9 {o o o o o o o o X} {...o o o o o o [X]}
4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
6.1 {X o o o X o o o} {[X] o o o [X] o o...}
6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
} {
do_execsql_test 1.$tn.1 {
DELETE FROM t1;
INSERT INTO t1 VALUES($doc, NULL);
SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X';
} [list $res]
do_execsql_test 1.$tn.2 {
DELETE FROM t1;
INSERT INTO t1 VALUES(NULL, $doc);
SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X';
} [list $res]
}
foreach {tn doc res} {
1.1 {X Y o o o o o} {[X Y] o o o o o}
1.2 {o X Y o o o o} {o [X Y] o o o o}
1.3 {o o X Y o o o} {o o [X Y] o o o}
1.4 {o o o X Y o o} {o o o [X Y] o o}
1.5 {o o o o X Y o} {o o o o [X Y] o}
1.6 {o o o o o X Y} {o o o o o [X Y]}
2.1 {X Y o o o o o o} {[X Y] o o o o o...}
2.2 {o X Y o o o o o} {o [X Y] o o o o...}
2.3 {o o X Y o o o o} {o o [X Y] o o o...}
2.4 {o o o X Y o o o} {...o o [X Y] o o o}
2.5 {o o o o X Y o o} {...o o o [X Y] o o}
2.6 {o o o o o X Y o} {...o o o o [X Y] o}
2.7 {o o o o o o X Y} {...o o o o o [X Y]}
3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
} {
do_execsql_test 2.$tn.1 {
DELETE FROM t1;
INSERT INTO t1 VALUES($doc, NULL);
SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X+Y';
} [list $res]
do_execsql_test 2.$tn.2 {
DELETE FROM t1;
INSERT INTO t1 VALUES(NULL, $doc);
SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X+Y';
} [list $res]
}
finish_test
test/permutations.test
+1
View File
@@ -226,6 +226,7 @@ test_suite "fts5" -prefix "" -description {
All FTS5 tests.
} -files {
fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test fts5ea.test
fts5af.test
}
test_suite "nofaultsim" -prefix "" -description {