diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index ad0cb36da9..e8b38f2ed3 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1276,6 +1276,7 @@ static int fts3InitVtab( p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bDescIdx = bDescIdx; + p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -3102,8 +3103,21 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); - sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + const int nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * p->mxLevel; + A += (A/2); + if( A>nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + } + sqlite3Fts3SegmentsClose(p); return rc; } @@ -3118,6 +3132,7 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index f84d15d196..abf5b76c97 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -196,11 +196,14 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ + u8 bAutoincrmerge; /* True if automerge=1 */ + int mxLevel; /* Maximum level seen on this transaction */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[36]; + sqlite3_stmt *aStmt[37]; char *zReadExprlist; char *zWriteExprlist; @@ -428,6 +431,7 @@ int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); void sqlite3Fts3SegmentsClose(Fts3Table *); +int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 @@ -479,6 +483,8 @@ struct Fts3MultiSegReader { int nDoclist; /* Size of aDoclist[] in bytes */ }; +int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 30f5e666eb..51f47d0106 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -72,6 +72,7 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; */ #define FTS_STAT_DOCTOTAL 0 #define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 /* ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic @@ -264,6 +265,7 @@ struct SegmentNode { #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle @@ -330,7 +332,7 @@ static int fts3SqlStmt( /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See -** function fts3Incrmerge() for details. */ +** function sqlite3Fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", @@ -362,7 +364,11 @@ static int fts3SqlStmt( /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ -/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC" +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -1868,6 +1874,35 @@ static int fts3WriteSegment( return rc; } +/* +** Update the Fts3Table.mxLevel field, if appropriate +*/ +static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){ + iLevel %= FTS3_SEGDIR_MAXLEVEL; + if( iLevel>p->mxLevel ) p->mxLevel = iLevel; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + /* ** Insert a record into the %_segdir table. */ @@ -1885,6 +1920,7 @@ static int fts3WriteSegdir( int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); + fts3UpdateMaxLevel(p, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); @@ -2184,6 +2220,7 @@ static int fts3SegWriterAdd( /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; /* Add the current term to the interior node tree. The term added to ** the interior tree must: @@ -2292,6 +2329,7 @@ static int fts3SegWriterFlush( rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } + p->nLeafAdd++; return rc; } @@ -2369,6 +2407,7 @@ static int fts3SegmentMaxLevel( ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); + fts3UpdateMaxLevel(p, *pnMax); } return sqlite3_reset(pStmt); } @@ -2999,11 +3038,28 @@ static int fts3SegmentMerge( int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + rc = sqlite3_reset(pStmt); + } + } return rc; } @@ -4472,7 +4528,7 @@ static int fts3IncrmergeHintLoad( ** nMin segments. Multiple merges might occur in an attempt to write the ** quota of nMerge leaf blocks. */ -static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ int bUseHint = 1; /* True if hint has not yet been attempted */ @@ -4578,6 +4634,19 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ return rc; } +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + /* ** Process statements of the form: ** @@ -4597,30 +4666,49 @@ static int fts3DoIncrmerge( const char *z = zParam; /* Read the first integer value */ - for(z=zParam; z[0]>='0' && z[0]<='9'; z++){ - nMerge = nMerge * 10 + (z[0] - '0'); - } + nMerge = fts3Getint(&z); /* If the first integer value is followed by a ',', read the second ** integer value. */ if( z[0]==',' && z[1]!='\0' ){ z++; - nMin = 0; - while( z[0]>='0' && z[0]<='9' ){ - nMin = nMin * 10 + (z[0] - '0'); - z++; - } + nMin = fts3Getint(&z); } if( z[0]!='\0' || nMin<2 ){ rc = SQLITE_ERROR; }else{ - rc = fts3Incrmerge(p, nMerge, nMin); + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); sqlite3Fts3SegmentsClose(p); } return rc; } +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + p->bAutoincrmerge = fts3Getint(&zParam)!=0; + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc;; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + + /* ** Handle a 'special' INSERT of the form: ** @@ -4642,6 +4730,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoRebuild(p); }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); diff --git a/manifest b/manifest index a6dafc26fe..fbaa006611 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sspurious\sSQLITE_CONSTRAINT\serror\sthat\smay\sbe\sreturned\sby\san\sincr-merge\soperation. -D 2012-03-23T18:26:11.608 +C Merge\sauto-incr-merge\swith\sincr-merge\sbranch. +D 2012-03-24T14:45:59.366 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c c47bb6fe7a6bde82cbdf4c504302221710699d64 +F ext/fts3/fts3.c 442f0ccf5c1a6a63844b9712b6f66bf8adb603b6 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 5fe1651db88206ca68f421cf62bb8501395677a3 +F ext/fts3/fts3Int.h f65df0fd76617cd43cb9795e5eec27e90ee4cbab F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c e3b750530132b3237aed3e5f008845cc4d7865f9 +F ext/fts3/fts3_write.c c71f84711f189bc18a82560949ac1656318fd2c5 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -295,7 +295,7 @@ F test/backup_ioerr.test 40d208bc9224b666ee3ed423f49bc9062a36a9d0 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f -F test/bc_common.tcl df4c51ae0de0cb414a18027be2abb2bd7693ce7a +F test/bc_common.tcl 5c8689cc6d2fb44b7c0968ae4f85eb26d50022fa F test/between.test 16b1776c6323faadb097a52d673e8e3d8be7d070 F test/bigfile.test 82dfe93ee7eb9e2e05641afa2b39ffd947a92ff1 F test/bigfile2.test 852f948cb492aadab45b58f4d2f3b0832a115cb0 @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 1c72cecc6bf5be2a5c04ad6214a6bac22a29f860 -R c0998f3b8bc74543bb7322ba65e1257e +P ed7c17ea165f6348506bd23ebc58c427bb65d697 0d841c957c6ec4afecb49504177c6279e09d7012 +R bf494068b52a47b15eebd4119e5e6c0c U dan -Z 3e48c8e93d6fd57a390b8a97532e70ea +Z a37d5b74ba1d4a69b01095005645cefa diff --git a/manifest.uuid b/manifest.uuid index 0886bc8716..6bd6c7d918 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed7c17ea165f6348506bd23ebc58c427bb65d697 \ No newline at end of file +1c68687ab6d05b100191663820e7d82377d52445 \ No newline at end of file diff --git a/test/bc_common.tcl b/test/bc_common.tcl index 8bba0245a5..eb9b6db9d3 100644 --- a/test/bc_common.tcl +++ b/test/bc_common.tcl @@ -17,7 +17,7 @@ proc bc_find_binaries {zCaption} { if {[llength $binaries]==0} { puts "WARNING: No historical binaries to test against." - puts "WARNING: Omitting backwards-compatibility tests $zFile" + puts "WARNING: Omitting backwards-compatibility tests" } foreach bin $binaries { @@ -70,7 +70,3 @@ proc do_all_bc_test {script} { uplevel [list do_bc_test $bin $script] } } - - - -