paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Instead of a temporary b-tree, use a linked-list and merge-sort to sort records in main memory in vdbesort.c.

Browse Source 
FossilOrigin-Name: 7769fb988d9be0f2d8129aaac19620ac88f9b4a6
This commit is contained in:
dan
2011-09-02 10:31:11 +00:00
parent 1c9d835d49
commit 5134d1357e
12 changed files with 423 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vdbesort.c
+306 -132
View File
@@ -21,6 +21,7 @@
#ifndef SQLITE_OMIT_MERGE_SORT
typedef struct VdbeSorterIter VdbeSorterIter;
typedef struct SorterRecord SorterRecord;
/*
** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES:
@@ -92,8 +93,7 @@ typedef struct VdbeSorterIter VdbeSorterIter;
** being merged (rounded up to the next power of 2).
*/
struct VdbeSorter {
int nWorking; /* Start a new b-tree after this many pages */
int nBtree; /* Current size of b-tree contents as PMA */
int nInMemory; /* Current size of b-tree contents as PMA */
int nTree; /* Used size of aTree/aIter (power of 2) */
VdbeSorterIter *aIter; /* Array of iterators to merge */
int *aTree; /* Current state of incremental merge */
@@ -101,6 +101,9 @@ struct VdbeSorter {
i64 iReadOff; /* Current read offset within file pTemp1 */
sqlite3_file *pTemp1; /* PMA file 1 */
int nPMA; /* Number of PMAs stored in pTemp1 */
SorterRecord *pRecord; /* Head of in-memory record list */
int nLimit1; /* Minimum PMA size, in bytes */
int nLimit2; /* Maximum PMA size, in bytes */
};
/*
@@ -117,6 +120,17 @@ struct VdbeSorterIter {
u8 *aKey; /* Pointer to current key */
};
/*
** A structure to store a single record. All in-memory records are connected
** together into a linked list headed at VdbeSorter.pRecord using the
** SorterRecord.pNext pointer.
*/
struct SorterRecord {
void *pVal;
int nVal;
SorterRecord *pNext;
};
/* Minimum allowable value for the VdbeSorter.nWorking variable */
#define SORTER_MIN_WORKING 10
@@ -275,6 +289,50 @@ static int vdbeSorterIterInit(
return rc;
}
/*
** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2,
** size nKey2 bytes). Argument pKeyInfo supplies the collation functions
** used by the comparison. If an error occurs, return an SQLite error code.
** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive
** value, depending on whether key1 is smaller, equal to or larger than key2.
**
** If the bOmitRowid argument is non-zero, assume both keys end in a rowid
** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid
** is true and key1 contains even a single NULL value, it is considered to
** be less than key2. Even if key2 also contains NULL values.
*/
static int vdbeSorterCompare(
KeyInfo *pKeyInfo, /* Collation functions to use in comparison */
int bOmitRowid, /* Ignore rowid field at end of keys */
void *pKey1, int nKey1, /* Left side of comparison */
void *pKey2, int nKey2, /* Right side of comparison */
int *pRes /* OUT: Result of comparison */
){
char aSpace[150];
UnpackedRecord *r2;
int i;
r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, sizeof(aSpace));
if( r2==0 ) return SQLITE_NOMEM;
if( bOmitRowid ){
for(i=0; i<r2->nField-1; i++){
if( r2->aMem[i].flags & MEM_Null ){
*pRes = -1;
sqlite3VdbeDeleteUnpackedRecord(r2);
return SQLITE_OK;
}
}
r2->flags |= UNPACKED_PREFIX_MATCH;
r2->nField--;
assert( r2->nField>0 );
}
*pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2);
sqlite3VdbeDeleteUnpackedRecord(r2);
return SQLITE_OK;
}
/*
** This function is called to compare two iterator keys when merging
** multiple b-tree segments. Parameter iOut is the index of the aTree[]
@@ -306,20 +364,16 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){
}else if( p2->pFile==0 ){
iRes = i1;
}else{
char aSpace[150];
UnpackedRecord *r1;
r1 = sqlite3VdbeRecordUnpack(
pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace)
int res;
int rc = vdbeSorterCompare(
pCsr->pKeyInfo, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res
);
if( r1==0 ) return SQLITE_NOMEM;
if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){
if( rc!=SQLITE_OK ) return rc;
if( res<=0 ){
iRes = i1;
}else{
iRes = i2;
}
sqlite3VdbeDeleteUnpackedRecord(r1);
}
pSorter->aTree[iOut] = iRes;
@@ -330,9 +384,32 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){
** Initialize the temporary index cursor just opened as a sorter cursor.
*/
int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
assert( pCsr->pKeyInfo && pCsr->pBt );
int pgsz; /* Page size of main database */
assert( pCsr->pKeyInfo && pCsr->pBt==0 );
pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter));
return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM);
if( pCsr->pSorter==0 ){
return SQLITE_NOMEM;
}
pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
pCsr->pSorter->nLimit1 = 10 * pgsz;
pCsr->pSorter->nLimit2 = db->aDb[0].pSchema->cache_size * pgsz;
return SQLITE_OK;
}
/*
** Free the list of sorted records starting at pRecord.
*/
static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){
SorterRecord *p;
SorterRecord *pNext;
for(p=pRecord; p; p=pNext){
pNext = p->pNext;
sqlite3DbFree(db, p->pVal);
sqlite3DbFree(db, p);
}
}
/*
@@ -351,6 +428,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
if( pSorter->pTemp1 ){
sqlite3OsCloseFree(pSorter->pTemp1);
}
vdbeSorterRecordFree(db, pSorter->pRecord);
sqlite3DbFree(db, pSorter);
pCsr->pSorter = 0;
}
@@ -370,10 +448,128 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){
);
}
/*
** Attemp to merge the two sorted lists p1 and p2 into a single list. If no
** error occurs set *ppOut to the head of the new list and return SQLITE_OK.
*/
static int vdbeSorterMerge(
sqlite3 *db, /* Database handle */
KeyInfo *pKeyInfo, /* Collation functions to use in comparison */
SorterRecord *p1, /* First list to merge */
SorterRecord *p2, /* Second list to merge */
SorterRecord **ppOut /* OUT: Head of merged list */
){
int rc = SQLITE_OK;
SorterRecord *pFinal = 0;
SorterRecord **pp = &pFinal;
while( p1 || p2 ){
if( p1==0 ){
*pp = p2;
p2 = 0;
}else if( p2==0 ){
*pp = p1;
p1 = 0;
}else{
int res;
rc = vdbeSorterCompare(
pKeyInfo, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res
);
if( rc!=SQLITE_OK ){
vdbeSorterRecordFree(db, p1);
vdbeSorterRecordFree(db, p2);
vdbeSorterRecordFree(db, pFinal);
pFinal = 0;
break;
}
if( res<=0 ){
*pp = p1;
pp = &p1->pNext;
p1 = p1->pNext;
}else{
*pp = p2;
pp = &p2->pNext;
p2 = p2->pNext;
}
*pp = 0;
}
}
*ppOut = pFinal;
return rc;
}
/*
** Write the current contents of the b-tree to a PMA. Return SQLITE_OK
** if successful, or an SQLite error code otherwise.
** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK
** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error
** occurs.
*/
static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){
int rc = SQLITE_OK;
int i;
SorterRecord **aSlot;
SorterRecord *p;
VdbeSorter *pSorter = pCsr->pSorter;
KeyInfo *pKeyInfo = pCsr->pKeyInfo;
aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *));
if( !aSlot ){
return SQLITE_NOMEM;
}
p = pSorter->pRecord;
while( p ){
SorterRecord *pNext = p->pNext;
p->pNext = 0;
for(i=0; rc==SQLITE_OK && aSlot[i]; i++){
rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p);
aSlot[i] = 0;
}
if( rc!=SQLITE_OK ){
vdbeSorterRecordFree(db, pNext);
break;
}
aSlot[i] = p;
p = pNext;
}
p = 0;
for(i=0; i<64; i++){
if( rc==SQLITE_OK ){
rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p);
}else{
vdbeSorterRecordFree(db, aSlot[i]);
}
}
pSorter->pRecord = p;
#if 0
{
SorterRecord *pTmp1 = 0;
SorterRecord *pTmp2;
for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){
if( pTmp1 ){
int res;
rc = vdbeSorterCompare(pKeyInfo,
0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res
);
assert( rc!=SQLITE_OK || res<0 );
}
pTmp1 = pTmp2;
}
}
#endif
if( rc!=SQLITE_OK ){
}
sqlite3_free(aSlot);
return rc;
}
/*
** Write the current contents of the in-memory linked-list to a PMA. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
**
** The format of a PMA is:
**
@@ -384,19 +580,19 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){
** Each record consists of a varint followed by a blob of data (the
** key). The varint is the number of bytes in the blob of data.
*/
static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){
int rc = SQLITE_OK; /* Return code */
VdbeSorter *pSorter = pCsr->pSorter;
int res = 0;
/* sqlite3BtreeFirst() cannot fail because sorter btrees are always held
** in memory and so an I/O error is not possible. */
rc = sqlite3BtreeFirst(pCsr->pCursor, &res);
if( NEVER(rc!=SQLITE_OK) || res ) return rc;
assert( pSorter->nBtree>0 );
if( pSorter->nInMemory==0 ){
assert( pSorter->pRecord==0 );
return rc;
}
rc = vdbeSorterSort(db, pCsr);
/* If the first temporary PMA file has not been opened, open it now. */
if( pSorter->pTemp1==0 ){
if( rc==SQLITE_OK && pSorter->pTemp1==0 ){
rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1);
assert( rc!=SQLITE_OK || pSorter->pTemp1 );
assert( pSorter->iWriteOff==0 );
@@ -404,129 +600,89 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
}
if( rc==SQLITE_OK ){
i64 iWriteOff = pSorter->iWriteOff;
void *aMalloc = 0; /* Array used to hold a single record */
int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */
i64 iOff = pSorter->iWriteOff;
SorterRecord *p;
SorterRecord *pNext = 0;
pSorter->nPMA++;
for(
rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff);
rc==SQLITE_OK && res==0;
rc = sqlite3BtreeNext(pCsr->pCursor, &res)
){
i64 nKey; /* Size of this key in bytes */
rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff);
for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){
pNext = p->pNext;
rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff);
/* Write the size of the record in bytes to the output file */
(void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey);
rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff);
/* Make sure the aMalloc[] buffer is large enough for the record */
if( rc==SQLITE_OK && nKey>nMalloc ){
aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey);
if( !aMalloc ){
rc = SQLITE_NOMEM;
}else{
nMalloc = nKey;
}
}
/* Write the record itself to the output file */
if( rc==SQLITE_OK ){
/* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */
rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
if( ALWAYS(rc==SQLITE_OK) ){
rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
iWriteOff += nKey;
}
rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff);
iOff += p->nVal;
}
if( rc!=SQLITE_OK ) break;
sqlite3DbFree(db, p->pVal);
sqlite3DbFree(db, p);
}
/* This assert verifies that unless an error has occurred, the size of
** the PMA on disk is the same as the expected size stored in
** pSorter->nBtree. */
assert( rc!=SQLITE_OK || pSorter->nBtree==(
iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree)
** pSorter->nInMemory. */
assert( rc!=SQLITE_OK || pSorter->nInMemory==(
iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory)
));
pSorter->iWriteOff = iWriteOff;
sqlite3DbFree(db, aMalloc);
pSorter->iWriteOff = iOff;
pSorter->pRecord = p;
}
pSorter->nBtree = 0;
return rc;
}
/*
** This function is called on a sorter cursor by the VDBE before each row
** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in
** bytes, about to be inserted.
**
** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr
** is large enough, its contents are written to a sorted PMA on disk and the
** tree emptied. This prevents the b-tree (which must be small enough to
** fit entirely in the cache in order to support efficient inserts) from
** growing too large.
**
** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK.
** Add a record to the sorter.
*/
int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){
int rc = SQLITE_OK; /* Return code */
int sqlite3VdbeSorterWrite(
sqlite3 *db, /* Database handle */
VdbeCursor *pCsr, /* Sorter cursor */
Mem *pVal /* Memory cell containing record */
){
VdbeSorter *pSorter = pCsr->pSorter;
if( pSorter ){
Pager *pPager = sqlite3BtreePager(pCsr->pBt);
int nPage; /* Current size of temporary file in pages */
int rc;
SorterRecord *pNew;
/* Sorters never spill to disk */
assert( sqlite3PagerFile(pPager)->pMethods==0 );
assert( pSorter );
pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n;
/* Determine how many pages the temporary b-tree has grown to */
sqlite3PagerPagecount(pPager, &nPage);
/* If pSorter->nWorking is still zero, but the temporary file has been
** created in the file-system, then the most recent insert into the
** current b-tree segment probably caused the cache to overflow (it is
** also possible that sqlite3_release_memory() was called). So set the
** size of the working set to a little less than the current size of the
** file in pages. */
if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){
pSorter->nWorking = nPage-5;
if( pSorter->nWorking<SORTER_MIN_WORKING ){
pSorter->nWorking = SORTER_MIN_WORKING;
}
pNew = (SorterRecord *)sqlite3DbMallocZero(db, sizeof(SorterRecord));
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3VdbeMemMakeWriteable(pVal);
if( rc==SQLITE_OK ){
pNew->pVal = pVal->z;
pNew->nVal = pVal->n;
pVal->zMalloc = 0;
sqlite3VdbeMemSetNull(pVal);
pNew->pNext = pSorter->pRecord;
pSorter->pRecord = pNew;
}else{
sqlite3DbFree(db, pNew);
rc = SQLITE_NOMEM;
}
/* If the number of pages used by the current b-tree segment is greater
** than the size of the working set (VdbeSorter.nWorking), start a new
** segment b-tree. */
if( pSorter->nWorking && nPage>=pSorter->nWorking ){
BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */
int iRoot; /* Root page of new tree */
/* Copy the current contents of the b-tree into a PMA in sorted order.
** Close the currently open b-tree cursor. */
rc = vdbeSorterBtreeToPMA(db, pCsr);
sqlite3BtreeCloseCursor(p);
if( rc==SQLITE_OK ){
rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0);
#ifdef SQLITE_DEBUG
sqlite3PagerPagecount(pPager, &nPage);
assert( rc!=SQLITE_OK || nPage==1 );
#endif
}
if( rc==SQLITE_OK ){
rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY);
}
if( rc==SQLITE_OK ){
assert( iRoot==2 );
rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p);
}
}
pSorter->nBtree += sqlite3VarintLen(nKey) + nKey;
}
/* See if the contents of the sorter should now be written out. They
** are written out when either of the following are true:
**
** * The total memory allocated for the in-memory list is greater
** than (page-size * cache-size), or
**
** * The total memory allocated for the in-memory list is greater
** than (page-size * 10) and sqlite3HeapNearlyFull() returns true.
*/
if( rc==SQLITE_OK && (
(pSorter->nInMemory>pSorter->nLimit2)
|| (pSorter->nInMemory>pSorter->nLimit1 && sqlite3HeapNearlyFull())
)){
rc = vdbeSorterListToPMA(db, pCsr);
pSorter->nInMemory = 0;
}
return rc;
}
@@ -577,8 +733,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
assert( pSorter );
/* Write the current b-tree to a PMA. Close the b-tree cursor. */
rc = vdbeSorterBtreeToPMA(db, pCsr);
sqlite3BtreeCloseCursor(pCsr->pCursor);
rc = vdbeSorterListToPMA(db, pCsr);
if( rc!=SQLITE_OK ) return rc;
if( pSorter->nPMA==0 ){
*pbEof = 1;
@@ -692,14 +847,7 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){
VdbeSorterIter *pIter;
pIter = &pSorter->aIter[ pSorter->aTree[1] ];
/* Coverage testing note: As things are currently, this call will always
** succeed. This is because the memory cell passed by the VDBE layer
** happens to be the same one as was used to assemble the keys before they
** were passed to the sorter - meaning it is always large enough for the
** largest key. But this could change very easily, so we leave the call
** to sqlite3VdbeMemGrow() in. */
if( NEVER(sqlite3VdbeMemGrow(pOut, pIter->nKey, 0)) ){
if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){
return SQLITE_NOMEM;
}
pOut->n = pIter->nKey;
@@ -709,4 +857,30 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){
return SQLITE_OK;
}
/*
** Compare the key in memory cell pVal with the key that the sorter cursor
** passed as the first argument currently points to. For the purposes of
** the comparison, ignore the rowid field at the end of each record.
**
** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM).
** Otherwise, set *pRes to a negative, zero or positive value if the
** key in pVal is smaller than, equal to or larger than the current sorter
** key.
*/
int sqlite3VdbeSorterCompare(
VdbeCursor *pCsr, /* Sorter cursor */
Mem *pVal, /* Value to compare to current sorter key */
int *pRes /* OUT: Result of comparison */
){
int rc;
VdbeSorter *pSorter = pCsr->pSorter;
VdbeSorterIter *pIter;
pIter = &pSorter->aIter[ pSorter->aTree[1] ];
rc = vdbeSorterCompare(pCsr->pKeyInfo, 1,
pVal->z, pVal->n, pIter->aKey, pIter->nKey, pRes
);
assert( rc!=SQLITE_OK || *pRes<=0 );
return rc;
}
#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */