Add an experimental integrity-check function to FTS.
FossilOrigin-Name: 40fc8804743dfb005991e9c5ef7b0ebcb3c2e731
This commit is contained in:
+2
-6
@@ -2691,7 +2691,7 @@ static int fts3SegReaderCursor(
|
||||
*/
|
||||
int sqlite3Fts3SegReaderCursor(
|
||||
Fts3Table *p, /* FTS3 table handle */
|
||||
int iLangid,
|
||||
int iLangid, /* Language-id to search */
|
||||
int iIndex, /* Index to search (from 0 to p->nIndex-1) */
|
||||
int iLevel, /* Level of segments to scan */
|
||||
const char *zTerm, /* Term to query for */
|
||||
@@ -2709,12 +2709,7 @@ int sqlite3Fts3SegReaderCursor(
|
||||
assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
|
||||
assert( isPrefix==0 || isScan==0 );
|
||||
|
||||
/* "isScan" is only set to true by the ft4aux module, an ordinary
|
||||
** full-text tables. */
|
||||
assert( isScan==0 || p->aIndex==0 );
|
||||
|
||||
memset(pCsr, 0, sizeof(Fts3MultiSegReader));
|
||||
|
||||
return fts3SegReaderCursor(
|
||||
p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
|
||||
);
|
||||
@@ -5206,6 +5201,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
** Return SQLITE_CORRUPT_VTAB.
|
||||
*/
|
||||
|
||||
@@ -134,6 +134,7 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
|
||||
typedef short int i16; /* 2-byte (or larger) signed integer */
|
||||
typedef unsigned int u32; /* 4-byte unsigned integer */
|
||||
typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */
|
||||
typedef sqlite3_int64 i64; /* 8-byte signed integer */
|
||||
|
||||
/*
|
||||
** Macro used to suppress compiler warnings for unused parameters.
|
||||
|
||||
@@ -4706,6 +4706,211 @@ static int fts3DoAutoincrmerge(
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Return a 64-bit checksum for the FTS index entry specified by the
|
||||
** arguments to this function.
|
||||
*/
|
||||
static i64 fts3ChecksumEntry(
|
||||
const char *zTerm, /* Pointer to buffer containing term */
|
||||
int nTerm, /* Size of zTerm in bytes */
|
||||
int iLangid, /* Language id for current row */
|
||||
int iIndex, /* Index (0..Fts3Table.nIndex-1) */
|
||||
i64 iDocid, /* Docid for current row. */
|
||||
int iCol, /* Column number */
|
||||
int iPos /* Position */
|
||||
){
|
||||
int i;
|
||||
i64 ret = iDocid;
|
||||
|
||||
ret += (ret<<3) + iLangid;
|
||||
ret += (ret<<3) + iIndex;
|
||||
ret += (ret<<3) + iCol;
|
||||
ret += (ret<<3) + iPos;
|
||||
for(i=0; i<nTerm; i++) ret += (ret<<3) + zTerm[i];
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
** Return a checksum of all entries in the FTS index that correspond to
|
||||
** language id iLangid. The checksum is calculated by XORing the checksums
|
||||
** of each individual entry (see fts3ChecksumEntry()) together.
|
||||
**
|
||||
** If successful, the checksum value is returned and *pRc set to SQLITE_OK.
|
||||
** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The
|
||||
** return value is undefined in this case.
|
||||
*/
|
||||
static i64 fts3ChecksumIndex(
|
||||
Fts3Table *p, /* FTS3 table handle */
|
||||
int iLangid, /* Language id to return cksum for */
|
||||
int iIndex, /* Index to cksum (0..p->nIndex-1) */
|
||||
int *pRc /* OUT: Return code */
|
||||
){
|
||||
Fts3SegFilter filter;
|
||||
Fts3MultiSegReader csr;
|
||||
int rc;
|
||||
i64 cksum = 0;
|
||||
|
||||
assert( *pRc==SQLITE_OK );
|
||||
|
||||
memset(&filter, 0, sizeof(filter));
|
||||
memset(&csr, 0, sizeof(csr));
|
||||
filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
|
||||
filter.flags |= FTS3_SEGMENT_SCAN;
|
||||
|
||||
rc = sqlite3Fts3SegReaderCursor(
|
||||
p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr
|
||||
);
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){
|
||||
char *pCsr = csr.aDoclist;
|
||||
char *pEnd = &pCsr[csr.nDoclist];
|
||||
|
||||
i64 iDocid = 0;
|
||||
i64 iCol = 0;
|
||||
i64 iPos = 0;
|
||||
|
||||
pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
|
||||
while( pCsr<pEnd ){
|
||||
i64 iVal = 0;
|
||||
pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
|
||||
if( pCsr<pEnd ){
|
||||
if( iVal==0 || iVal==1 ){
|
||||
iCol = 0;
|
||||
iPos = 0;
|
||||
if( iVal ){
|
||||
pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
|
||||
}else{
|
||||
pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
|
||||
iDocid += iVal;
|
||||
}
|
||||
}else{
|
||||
iPos += (iVal - 2);
|
||||
cksum = cksum ^ fts3ChecksumEntry(
|
||||
csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, iCol, iPos
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sqlite3Fts3SegReaderFinish(&csr);
|
||||
|
||||
*pRc = rc;
|
||||
return cksum;
|
||||
}
|
||||
|
||||
/*
|
||||
** Check if the contents of the FTS index match the current contents of the
|
||||
** content table. If no error occurs and the contents do match, set *pbOk
|
||||
** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk
|
||||
** to false before returning.
|
||||
**
|
||||
** If an error occurs (e.g. an OOM or IO error), return an SQLite error
|
||||
** code. The final value of *pbOk is undefined in this case.
|
||||
*/
|
||||
static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
|
||||
int rc = SQLITE_OK; /* Return code */
|
||||
i64 cksum1 = 0; /* Checksum based on FTS index contents */
|
||||
i64 cksum2 = 0; /* Checksum based on %_content contents */
|
||||
sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */
|
||||
|
||||
/* This block calculates the checksum according to the FTS index. */
|
||||
rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
|
||||
if( rc==SQLITE_OK ){
|
||||
int rc2;
|
||||
sqlite3_bind_int(pAllLangid, 1, p->nIndex);
|
||||
while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
|
||||
int iLangid = sqlite3_column_int(pAllLangid, 0);
|
||||
int i;
|
||||
for(i=0; i<p->nIndex; i++){
|
||||
cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc);
|
||||
}
|
||||
}
|
||||
rc2 = sqlite3_reset(pAllLangid);
|
||||
if( rc==SQLITE_OK ) rc = rc2;
|
||||
}
|
||||
|
||||
/* This block calculates the checksum according to the %_content table */
|
||||
rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
|
||||
if( rc==SQLITE_OK ){
|
||||
sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
|
||||
sqlite3_stmt *pStmt = 0;
|
||||
char *zSql;
|
||||
|
||||
zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
|
||||
if( !zSql ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}else{
|
||||
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
|
||||
sqlite3_free(zSql);
|
||||
}
|
||||
|
||||
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
|
||||
i64 iDocid = sqlite3_column_int64(pStmt, 0);
|
||||
int iLang = langidFromSelect(p, pStmt);
|
||||
int iCol;
|
||||
|
||||
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
|
||||
const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
|
||||
int nText = sqlite3_column_bytes(pStmt, iCol+1);
|
||||
sqlite3_tokenizer_cursor *pT = 0;
|
||||
|
||||
rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT);
|
||||
while( rc==SQLITE_OK ){
|
||||
char const *zToken; /* Buffer containing token */
|
||||
int nToken; /* Number of bytes in token */
|
||||
int iDum1, iDum2; /* Dummy variables */
|
||||
int iPos; /* Position of token in zText */
|
||||
|
||||
rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos);
|
||||
if( rc==SQLITE_OK ){
|
||||
int i;
|
||||
cksum2 = cksum2 ^ fts3ChecksumEntry(
|
||||
zToken, nToken, iLang, 0, iDocid, iCol, iPos
|
||||
);
|
||||
for(i=1; i<p->nIndex; i++){
|
||||
if( p->aIndex[i].nPrefix<=nToken ){
|
||||
cksum2 = cksum2 ^ fts3ChecksumEntry(
|
||||
zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if( pT ) pModule->xClose(pT);
|
||||
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
|
||||
}
|
||||
}
|
||||
|
||||
sqlite3_finalize(pStmt);
|
||||
}
|
||||
|
||||
*pbOk = (cksum1==cksum2);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Run the integrity-check. If no error occurs and the current contents of
|
||||
** the FTS index are correct, return SQLITE_OK. Or, if the contents of the
|
||||
** FTS index are incorrect, return SQLITE_CORRUPT_VTAB.
|
||||
**
|
||||
** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite
|
||||
** error code.
|
||||
*/
|
||||
static int fts3DoIntegrityCheck(
|
||||
Fts3Table *p /* FTS3 table handle */
|
||||
){
|
||||
int rc;
|
||||
int bOk = 0;
|
||||
rc = fts3IntegrityCheck(p, &bOk);
|
||||
if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Handle a 'special' INSERT of the form:
|
||||
@@ -4726,6 +4931,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
|
||||
rc = fts3DoOptimize(p, 0);
|
||||
}else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){
|
||||
rc = fts3DoRebuild(p);
|
||||
}else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){
|
||||
rc = fts3DoIntegrityCheck(p);
|
||||
}else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){
|
||||
rc = fts3DoIncrmerge(p, &zVal[6]);
|
||||
}else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
|
||||
|
||||
Reference in New Issue
Block a user