diff --git a/ext/fts2/fts2.c b/ext/fts2/fts2.c index 48d0af55d6..b55443d3c3 100644 --- a/ext/fts2/fts2.c +++ b/ext/fts2/fts2.c @@ -152,7 +152,10 @@ ** than the first term encoded (or all terms if no term is encoded). ** Otherwise, for terms greater than or equal to pTerm[i] but less ** than pTerm[i+1], the subtree for that term will be rooted at -** iBlockid+i. +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). ** ** New data is spilled to a new interior node at the same height when ** the current node exceeds INTERIOR_MAX bytes (default 2048). @@ -3961,6 +3964,11 @@ typedef struct LeafWriter { DataBuffer term; /* previous encoded term */ DataBuffer data; /* encoding buffer */ + /* bytes of first term in the current node which distinguishes that + ** term from the last term of the previous node. + */ + int nTermDistinct; + InteriorWriter parentWriter; /* if we overflow */ int has_parent; } LeafWriter; @@ -4072,6 +4080,9 @@ static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter, n = getVarint32(pWriter->data.pData+iData+1, &nStartingTerm); pStartingTerm = pWriter->data.pData+iData+1+n; assert( pWriter->data.nData>iData+1+n+nStartingTerm ); + assert( pWriter->nTermDistinct>0 ); + assert( pWriter->nTermDistinct<=nStartingTerm ); + nStartingTerm = pWriter->nTermDistinct; if( pWriter->has_parent ){ interiorWriterAppend(&pWriter->parentWriter, @@ -4166,11 +4177,23 @@ static void leafWriterDestroy(LeafWriter *pWriter){ dataBufferDestroy(&pWriter->data); } -/* Encode a term into the leafWriter, delta-encoding as appropriate. */ -static void leafWriterEncodeTerm(LeafWriter *pWriter, - const char *pTerm, int nTerm){ +/* Encode a term into the leafWriter, delta-encoding as appropriate. +** Returns the length of the new term which distinguishes it from the +** previous term, which can be used to set nTermDistinct when a node +** boundary is crossed. +*/ +static int leafWriterEncodeTerm(LeafWriter *pWriter, + const char *pTerm, int nTerm){ char c[VARINT_MAX+VARINT_MAX]; - int n; + int n, nPrefix = 0; + + assert( nTerm>0 ); + while( nPrefixterm.nData && + pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){ + nPrefix++; + /* Failing this implies that the terms weren't in order. */ + assert( nPrefixdata.nData==0 ){ /* Encode the node header and leading term as: @@ -4187,21 +4210,13 @@ static void leafWriterEncodeTerm(LeafWriter *pWriter, ** varint(nSuffix) ** char pTermSuffix[nSuffix] */ - int nPrefix = 0; - - assert( nTerm>0 ); - while( nPrefixterm.nData && - pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){ - nPrefix++; - /* Failing this implies that the terms weren't in order. */ - assert( nPrefixdata, c, n, pTerm+nPrefix, nTerm-nPrefix); } dataBufferReplace(&pWriter->term, pTerm, nTerm); + + return nPrefix+1; } /* Used to avoid a memmove when a large amount of doclist data is in @@ -4238,10 +4253,13 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, DLReader *pReaders, int nReaders){ char c[VARINT_MAX+VARINT_MAX]; int iTermData = pWriter->data.nData, iDoclistData; - int i, nData, n, nActualData, nActual, rc; + int i, nData, n, nActualData, nActual, rc, nTermDistinct; ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData); - leafWriterEncodeTerm(pWriter, pTerm, nTerm); + nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm); + + /* Remember nTermDistinct if opening a new node. */ + if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct; iDoclistData = pWriter->data.nData; @@ -4283,6 +4301,8 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, if( iTermData>0 ){ rc = leafWriterInternalFlush(v, pWriter, 0, iTermData); if( rc!=SQLITE_OK ) return rc; + + pWriter->nTermDistinct = nTermDistinct; } /* Fix the encoded doclist length. */ @@ -4323,6 +4343,8 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, rc = leafWriterInternalFlush(v, pWriter, 0, iTermData); if( rc!=SQLITE_OK ) return rc; + pWriter->nTermDistinct = nTermDistinct; + /* Rebuild header using the current term */ n = putVarint(pWriter->data.pData, 0); n += putVarint(pWriter->data.pData+n, nTerm); diff --git a/manifest b/manifest index 4cdc1061a0..5d275357c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactoring\sgroundwork\sfor\scoming\swork\son\sinterior\snodes.\s\sChange\nLeafWriter\sto\suse\sempty\sdata\sbuffer\s(instead\sof\sempty\sterm)\sto\sdetect\nan\sempty\sblock.\s\sCode\sto\svalidate\sinterior\snodes.\s\sModerate\srevisions\nto\sleaf-node\sand\sdoclist\svalidation.\s\sRecast\sleafWriterStep()\sin\sterms\nof\sLeafWriterStepMerge().\s(CVS\s3512) -D 2006-11-17T21:12:16 +C Store\sminimal\sterms\sin\sinterior\snodes.\s\sWhenever\sthere's\sa\sbreak\nbetween\sleaf\snodes,\sinstead\sof\sstoring\sthe\sentire\sleftmost\sterm\sof\sthe\nrightmost\schild,\sstore\sonly\sthat\sportion\sof\sthe\sleftmost\sterm\nnecessary\sto\sdistinguish\sit\sfrom\sthe\srightmost\sterm\sof\sthe\sleftmost\nchild.\s(CVS\s3513) +D 2006-11-18T00:12:45 F Makefile.in 8e14898d41a53033ecb687d93c9cd5d109fb9ae3 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -33,7 +33,7 @@ F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9 F ext/fts2/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts2/fts2.c 57d8cd57ce18c3ce7b194b4810fe7e119ec7e6a3 +F ext/fts2/fts2.c 74a5db3f7f8e49dfa2a5d40e5fdece09bf23e5a8 F ext/fts2/fts2.h bbdab26d34f91974d5b9ade8b7836c140a7c4ce1 F ext/fts2/fts2_hash.c b3f22116d4ef0bc8f2da6e3fdc435c86d0951a9b F ext/fts2/fts2_hash.h e283308156018329f042816eb09334df714e105e @@ -421,7 +421,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P 9b6d413d751d962b67cb4e3a208efe61581cb822 -R ff81ed1c8b4721212823c87e00f2e6b9 +P f30771d5c7ef2b502af95d81a18796b75271ada4 +R ef562280fdedbce7ff0fcbb49811b62c U shess -Z 82e324f504b7a8a8d9f0a515f8d329aa +Z 674bc601b7eaf4b058181b0cf2143d33 diff --git a/manifest.uuid b/manifest.uuid index 7d116f7a68..f6e6ca6a8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f30771d5c7ef2b502af95d81a18796b75271ada4 \ No newline at end of file +f6e0b080dcfaf554b2c05df5e7d4db69d012fba3 \ No newline at end of file