paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

In fts4, store the total number of bytes of for all records in the table in the %_stat table.

Browse Source 
FossilOrigin-Name: 941647d121ac60e2eabc998cfe79b157fb918d7e
This commit is contained in:
dan
2010-10-27 10:55:54 +00:00
parent 61c7f59c10
commit 8c30711208
4 changed files with 53 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts3/fts3_write.c
+33 -21
View File
@@ -583,6 +583,7 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
return rc;
}
}
aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
}
return SQLITE_OK;
}
@@ -680,7 +681,7 @@ static int fts3DeleteAll(Fts3Table *p){
** (an integer) of a row about to be deleted. Remove all terms from the
** full-text index.
*/
static void fts3DeleteTerms(
static void fts3DeleteTerms(
int *pRC, /* Result code */
Fts3Table *p, /* The FTS table to delete from */
sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */
@@ -702,6 +703,7 @@ static void fts3DeleteTerms(
*pRC = rc;
return;
}
aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
}
rc = sqlite3_reset(pSelect);
@@ -1033,9 +1035,7 @@ int sqlite3Fts3SegReaderCost(
const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
a += sqlite3Fts3GetVarint(a, &nDoc);
while( a<pEnd ){
sqlite3_int64 nVarint;
a += sqlite3Fts3GetVarint(a, &nVarint);
nByte += nVarint;
a += sqlite3Fts3GetVarint(a, &nByte);
}
}
@@ -2517,16 +2517,26 @@ static void fts3InsertDocsize(
}
/*
** Update the 0 record of the %_stat table so that it holds a blob
** which contains the document count followed by the cumulative
** document sizes for all columns.
** Record 0 of the %_stat table contains a blob consisting of N varints,
** where N is the number of user defined columns in the fts3 table plus
** two. If nCol is the number of user defined columns, then values of the
** varints are set as follows:
**
** Varint 0: Total number of rows in the table.
**
** Varint 1..nCol: For each column, the total number of tokens stored in
** the column for all rows of the table.
**
** Varint 1+nCol: The total size, in bytes, of all text values in all
** columns of all rows of the table.
**
*/
static void fts3UpdateDocTotals(
int *pRC, /* The result code */
Fts3Table *p, /* Table being updated */
u32 *aSzIns, /* Size increases */
u32 *aSzDel, /* Size decreases */
int nChng /* Change in the number of documents */
int *pRC, /* The result code */
Fts3Table *p, /* Table being updated */
u32 *aSzIns, /* Size increases */
u32 *aSzDel, /* Size decreases */
int nChng /* Change in the number of documents */
){
char *pBlob; /* Storage for BLOB written into %_stat */
int nBlob; /* Size of BLOB written into %_stat */
@@ -2535,13 +2545,15 @@ static void fts3UpdateDocTotals(
int i; /* Loop counter */
int rc; /* Result code from subfunctions */
const int nStat = p->nColumn+2;
if( *pRC ) return;
a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) );
a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
if( a==0 ){
*pRC = SQLITE_NOMEM;
return;
}
pBlob = (char*)&a[p->nColumn+1];
pBlob = (char*)&a[nStat];
rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
@@ -2549,11 +2561,11 @@ static void fts3UpdateDocTotals(
return;
}
if( sqlite3_step(pStmt)==SQLITE_ROW ){
fts3DecodeIntArray(p->nColumn+1, a,
fts3DecodeIntArray(nStat, a,
sqlite3_column_blob(pStmt, 0),
sqlite3_column_bytes(pStmt, 0));
}else{
memset(a, 0, sizeof(u32)*(p->nColumn+1) );
memset(a, 0, sizeof(u32)*(nStat) );
}
sqlite3_reset(pStmt);
if( nChng<0 && a[0]<(u32)(-nChng) ){
@@ -2561,7 +2573,7 @@ static void fts3UpdateDocTotals(
}else{
a[0] += nChng;
}
for(i=0; i<p->nColumn; i++){
for(i=0; i<p->nColumn+1; i++){
u32 x = a[i+1];
if( x+aSzIns[i] < aSzDel[i] ){
x = 0;
@@ -2570,7 +2582,7 @@ static void fts3UpdateDocTotals(
}
a[i+1] = x;
}
fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob);
fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
@@ -2791,10 +2803,10 @@ int sqlite3Fts3UpdateMethod(
assert( p->pSegments==0 );
/* Allocate space to hold the change in document sizes */
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
if( aSzIns==0 ) return SQLITE_NOMEM;
aSzDel = &aSzIns[p->nColumn];
memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2);
aSzDel = &aSzIns[p->nColumn+1];
memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
/* If this is a DELETE or UPDATE operation, remove the old record. */
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
manifest
+7 -7
View File
@@ -1,5 +1,5 @@
C Structural\scoverage\stests\sfor\svdbeblob.c.\sIncluding\sexperimental\snew\sAPI\ssqlite3_blob_reopen().
D 2010-10-26T18:42:52
C In\sfts4,\sstore\sthe\stotal\snumber\sof\sbytes\sof\sfor\sall\srecords\sin\sthe\stable\sin\sthe\s%_stat\stable.
D 2010-10-27T10:55:54
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2c8cefd962eca0147132c7cf9eaa4bb24c656f3f
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -73,7 +73,7 @@ F ext/fts3/fts3_snippet.c 300c12b7f0a2a6ae0491bb2d00e2d5ff9c28f685
F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
F ext/fts3/fts3_write.c 1b9211904f8157ebca8e17034e1150f3653e6c3f
F ext/fts3/fts3_write.c 943216b144447a1fbadef373cbd2b7eb3fd17a65
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -433,7 +433,7 @@ F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
F test/fts3defer.test 1f75427fa35d7715595c91575b7c5d449fd28089
F test/fts3defer.test eab4f24c8402fb4e1e6aad44bcdfbe5bf42160b2
F test/fts3defer2.test 1a9f213ca79509b60d81460febc7e4e5b64af95c
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
@@ -879,7 +879,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P d1cc5c93f09c9092ec478c04e8d9a8b1f9c0cb04
R 1bc240279e5d7a2dcb9378295e259554
P 97c6b2616ddcce2337778c6ee88a973cc4fe999d
R 8864817f6f3963775cfdd60519bb3822
U dan
Z c7b142127578014c4915d0546d65c2ad
Z 890501f0776b5167d933a1fceff6796c
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
97c6b2616ddcce2337778c6ee88a973cc4fe999d
941647d121ac60e2eabc998cfe79b157fb918d7e
test/fts3defer.test
+12
View File
@@ -189,6 +189,15 @@ lappend data {*}{
"srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki"
}
proc add_empty_records {n} {
execsql BEGIN
for {set i 0} {$i < $n} {incr i} {
execsql { INSERT INTO t1 VALUES('') }
}
execsql COMMIT
}
#set e [list]
#foreach d $data {set e [concat $e $d]}
#puts [lsort -unique $e]
@@ -213,12 +222,14 @@ foreach {tn setup} {
set dmt_modes {0 1 2}
execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
add_empty_records 1000
execsql $zero_long_doclists
}
4 {
set dmt_modes 0
execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
add_empty_records 1000
execsql "INSERT INTO t1(t1) VALUES('optimize')"
execsql $zero_long_doclists
}
@@ -239,6 +250,7 @@ foreach {tn setup} {
do_select_test 1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
} {100}
if {$tn==3} breakpoint
do_select_test 1.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
} {7 70 98}