From 588400b861b2fa82a591b78a85cf3f460435ba8a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Sep 2014 05:00:25 +0000 Subject: [PATCH 01/10] Reduce the amount of memcpy() required by defragmentPage(). FossilOrigin-Name: 3edab9957cc7bb90b52fd40b02613c2cb03fc166 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/btree.c | 25 ++++++++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 131d77b3a9..574bfe2c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. -D 2014-09-26T02:41:05.726 +C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). +D 2014-09-27T05:00:25.096 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 59f03e421dad3cb6e27cc7d2393d3a7459be4b5e +F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 -R 32be19747fff5e8f2465eed0f224b45d +P 83913515830aa850f9e38406f9422d7e88dcab66 +R 66a1e1f00a844450677737824735607d +T *branch * defrag-opt +T *sym-defrag-opt * +T -sym-trunk * U drh -Z f56b9000203c19d0f3a8172e8374b279 +Z f60a1f2e4650c574e91ad245c0c67dee diff --git a/manifest.uuid b/manifest.uuid index b182c8c316..bf8d17f152 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file +3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 135b40139f..c4832b4ee9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1151,6 +1151,7 @@ static int defragmentPage(MemPage *pPage){ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ @@ -1160,15 +1161,13 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; + temp = 0; + src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; @@ -1187,7 +1186,7 @@ static int defragmentPage(MemPage *pPage){ } #endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); + size = cellSizePtr(pPage, &src[pc]); cbrk -= size; #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) if( cbrk=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -5955,13 +5962,13 @@ static void assemblePage( assert( pPage->nCell==0 ); assert( get2byteNotZero(&data[hdr+5])==nUsable ); - pCellptr = &pPage->aCellIdx[nCell*2]; + pCellptr = pPage->aCellIdx; cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ + for(i=0; i Date: Thu, 9 Oct 2014 19:35:37 +0000 Subject: [PATCH 02/10] Change the balance_nonroot() routine to reduce the amount of memcpy work that takes place. This is a work in progress. FossilOrigin-Name: 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 --- manifest | 21 +-- manifest.uuid | 2 +- src/btree.c | 494 +++++++++++++++++++++++++++----------------------- src/pager.c | 9 + src/pager.h | 2 + 5 files changed, 292 insertions(+), 236 deletions(-) diff --git a/manifest b/manifest index 574bfe2c8d..aeefa53238 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). -D 2014-09-27T05:00:25.096 +C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. +D 2014-10-09T19:35:37.452 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 +F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c caab007743821d96752597c9cfd7351654697b06 -F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 +F src/pager.c 0abcb0904a78d68b96357f360c6b160bcfc2a3e0 +F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 F src/parse.y b98772da2bb5415970085b707203f92569400aa8 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1200,10 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83913515830aa850f9e38406f9422d7e88dcab66 -R 66a1e1f00a844450677737824735607d -T *branch * defrag-opt -T *sym-defrag-opt * -T -sym-trunk * -U drh -Z f60a1f2e4650c574e91ad245c0c67dee +P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 +R 5fca1836b5a4d862df24682ecb47d048 +U dan +Z 8527330c8f275358262176fc962502e2 diff --git a/manifest.uuid b/manifest.uuid index bf8d17f152..bb422fd7dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file +29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c4832b4ee9..dbf002ecdb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5977,6 +5977,49 @@ static void assemblePage( pPage->nCell = (u16)nCell; } + +static void rebuildPage( + MemPage *pPg, /* Edit this page */ + int nRemove, /* Cells to remove from start of page */ + int nCell, /* Final number of cells on page */ + u8 **apCell, /* Array of nCell final cells */ + u16 *szCell /* Array of nCell cell sizes */ +){ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i; + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; + + i = get2byte(&aData[hdr+5]); + memcpy(&pTmp[i], &aData[i], usableSize - i); + pData = &aData[usableSize]; + + for(i=0; iaData && pCellnFree = (pData - pCellptr); + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6098,7 +6141,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 0 +#if 1 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6265,7 +6308,6 @@ static int balance_nonroot( int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ @@ -6276,6 +6318,13 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ + int aShiftLeft[NB+2]; + int aShiftRight[NB+2]; + u8 abDone[NB+2]; + Pgno aPgno[NB+2]; + u16 aPgFlags[NB+2]; + + memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -6384,12 +6433,10 @@ static int balance_nonroot( /* ** Allocate space for memory structures */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); szScratch = nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ + + pBt->pageSize; /* aSpace1 */ apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; @@ -6402,8 +6449,8 @@ static int balance_nonroot( /* ** Load pointers to all cells on sibling pages and the divider cells ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the divider cells - ** from pParent. + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied @@ -6419,15 +6466,7 @@ static int balance_nonroot( leafData = apOld[0]->intKeyLeaf; for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + MemPage *pOld = apOld[i]; limit = pOld->nCell+pOld->nOverflow; if( pOld->nOverflow>0 ){ @@ -6556,10 +6595,10 @@ static int balance_nonroot( assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); #endif - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 )); /* @@ -6582,6 +6621,7 @@ static int balance_nonroot( assert( i>0 ); rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); apNew[i] = pNew; nNew++; @@ -6595,135 +6635,223 @@ static int balance_nonroot( } } - /* Free any old pages that were not reused as new pages. - */ - while( ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + } + for(i=0; ipgno); + Pgno iMin = 0; + u16 flags = 0; + for(j=0; jiGt && (iMin==0 || iPgnoi ){ - MemPage *pT; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; + if( apNew[i]->pgno!=iMin ){ + apNew[i]->pDbPage->flags = flags; + sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + apNew[i]->pgno = iMin; } } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. - */ j = 0; for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); + /* At this point, "j" is the apCell[] index of the first cell currently + ** stored on page apNew[i]. Or, if apNew[i] was not one of the original + ** sibling pages, "j" should be set to nCell. Variable iFirst is set + ** to the apCell[] index of the first cell that will appear on the + ** page following this balancing operation. */ + int iFirst = (i==0 ? 0 : cntNew[i-1] + !leafData); /* new first cell */ + assert( inCell + apNew[i]->nOverflow; + aShiftRight[i] = cntNew[i] - j; + assert( i!=nOld-1 || j==nCell ); + if( jnOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( iaData; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int usableSize = pBt->usableSize; + int iNew = 0; + int iOld = 0; - assert( jleaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; - }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); + for(i=0; inCell + pOld->nOverflow + !leafData; + aOld = pOld->aData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } + + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow page), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( pNew->pgno!=aPgno[iOld] || pCell=&aOld[usableSize] ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( szCell[i]>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); } } - iOvflSpace += sz; - assert( sz<=pBt->maxLocal+23 ); - assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - - j++; - nxDiv++; } } - assert( j==nCell ); + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + btreeParseCellPtr(pNew, apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( szCell[j]==4 ){ + assert(leafCorrection==4); + sz = cellSizePtr(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** 1) If the aShiftLeft[] entry is less than 0, it is not safe to + ** update the page until the page to the left of the current page + ** (apNew[i-1]) has already been updated. + ** + ** 2) If the aShiftRight[] entry is less than 0, it is not safe to + ** update the page until the page to the right of the current page + ** (apNew[i+1]) has already been updated. + ** + ** If neither of the above apply, the page is safe to update. + */ + assert( aShiftRight[nNew-1]>=0 && aShiftLeft[0]==0 ); + for(i=0; i=nNew ? i-nNew : nNew-1-i); + if( abDone[iPg]==0 + && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) + && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + ){ + MemPage *pNew = apNew[iPg]; + int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); + rebuildPage(pNew, + aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, + cntNew[iPg] - iLeft, + &apCell[iLeft], + &szCell[iLeft] + ); + abDone[iPg] = 1; + assert( pNew->nOverflow==0 ); + assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + } + } + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + assert( nOld>0 ); assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); - } if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ /* The root page of the b-tree now contains no cells. The only sibling @@ -6746,116 +6874,36 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aiOvfl[0]; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); - assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jpgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); } + } - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } - } + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, nCell)); -#if 0 + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); ptrmapCheckPages(&pParent, 1); -#endif } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); +#endif /* ** Cleanup before returning. diff --git a/src/pager.c b/src/pager.c index 79bfe15f10..e68d147de5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6835,6 +6835,14 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } + +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ + PgHdr *pPg = (PgHdr*)pPage; + assert( pPg->flags & PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) ); + sqlite3PcacheMove(pPg, iNew); +} + #endif /* @@ -7235,4 +7243,5 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index c9ca8553b9..f3a04bbca6 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,6 +188,8 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +void sqlite3PagerRekey(DbPage*, Pgno); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif From 09c6840153697ea6679650b65dfa50bc15cef536 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 11 Oct 2014 20:00:24 +0000 Subject: [PATCH 03/10] Attempt to further reduce memcpy() in balance_nonroot(). FossilOrigin-Name: fec849dcca3aead2bc2d4ecffeda750684d32fb0 --- manifest | 12 ++-- manifest.uuid | 2 +- src/btree.c | 185 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index aeefa53238..7e782aa82c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. -D 2014-10-09T19:35:37.452 +C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). +D 2014-10-11T20:00:24.552 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 +F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 -R 5fca1836b5a4d862df24682ecb47d048 +P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 +R 3235ac37769ff059d7c714947e8a11dd U dan -Z 8527330c8f275358262176fc962502e2 +Z 17fdfd7faf788bbb79d2ec9941c17025 diff --git a/manifest.uuid b/manifest.uuid index bb422fd7dd..006b3c0008 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file +fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index dbf002ecdb..13ec912748 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5980,10 +5980,9 @@ static void assemblePage( static void rebuildPage( MemPage *pPg, /* Edit this page */ - int nRemove, /* Cells to remove from start of page */ int nCell, /* Final number of cells on page */ - u8 **apCell, /* Array of nCell final cells */ - u16 *szCell /* Array of nCell cell sizes */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ u8 * const aData = pPg->aData; /* Pointer to data for pPg */ @@ -6020,6 +6019,129 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static void editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + + if( 1 ){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nFree = pPg->nFree; /* Free bytes on pPg */ + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nShift++; + } + } + nCell -= nShift; + memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + } + for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nCell--; + } + } + pData = &aData[get2byte(&aData[hdr+5])]; + if( pDataaCellIdx; + memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); + for(i=iNew; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + int sz = szCell[iCell+iNew]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + pData -= sz; + if( pDataaCellIdx[nCell*2]; + for(i=iNew+nCell; i<(iNew+nNew); i++){ + pData -= szCell[i]; + if( pDatanFree = nFree; + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } +#endif + +#if 0 + printf("EDIT\n"); +#endif + + return; + } + + editpage_fail: +#if 0 + printf("REBUILD\n"); +#endif + /* Unable to edit this page. Rebuild it from scratch instead. */ + rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6312,6 +6434,7 @@ static int balance_nonroot( u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ int szNew[NB+2]; /* Combined size of cells place on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ @@ -6487,6 +6610,7 @@ static int balance_nonroot( nCell++; } } + cntOld[i] = nCell; if( i %d@%d", apNew[i]->pgno, + pNew->nCell+pNew->nOverflow, j, + cntNew[i] - iFirst, iFirst + ); + for(iCell=iFirst; iCellaData + || apCell[iCell]>=&pNew->aData[pBt->usableSize] + ){ + nCta += szCell[iCell]; + } + } + nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); + nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); + printf(" cta=%d free=%d\n", nCta, nFree); + if( i==(nNew-1) ){ + printf("-----\n"); + fflush(stdout); + } +#endif + assert( inCell + apNew[i]->nOverflow; @@ -6835,17 +6987,23 @@ static int balance_nonroot( && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) && (aShiftRight[iPg]>=0 || abDone[iPg+1]) ){ - MemPage *pNew = apNew[iPg]; - int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); - rebuildPage(pNew, - aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, - cntNew[iPg] - iLeft, - &apCell[iLeft], - &szCell[iLeft] - ); + int iNew; + int iOld; + int nNewCell; + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnOverflow==0 ); - assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); } } assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); @@ -6869,6 +7027,7 @@ static int balance_nonroot( ** is important if the parent page happens to be page 1 of the database ** image. */ assert( nNew==1 ); + defragmentPage(apNew[0]); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) ); From d7b545bbcb2225a33e8c8c5192ab11416a855999 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Oct 2014 18:03:27 +0000 Subject: [PATCH 04/10] Further work on balance_nonroot(). FossilOrigin-Name: 6594f9b420e2fa642737722ff8521f756ecef227 --- manifest | 12 +-- manifest.uuid | 2 +- src/btree.c | 265 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 180 insertions(+), 99 deletions(-) diff --git a/manifest b/manifest index 7e782aa82c..9f25f390d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). -D 2014-10-11T20:00:24.552 +C Further\swork\son\sbalance_nonroot(). +D 2014-10-13T18:03:27.743 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 +F src/btree.c 63211ca1d4ae867eede39a37901aec4746d904a7 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 -R 3235ac37769ff059d7c714947e8a11dd +P fec849dcca3aead2bc2d4ecffeda750684d32fb0 +R 79e5b5564c934fc68257d82dc897cefd U dan -Z 17fdfd7faf788bbb79d2ec9941c17025 +Z ae5b74eee28666050af6baaa639de56c diff --git a/manifest.uuid b/manifest.uuid index 006b3c0008..fcba485229 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file +6594f9b420e2fa642737722ff8521f756ecef227 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 13ec912748..7c9a7228de 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6009,7 +6009,7 @@ static void rebuildPage( assert( szCell[i]==cellSizePtr(pPg, pCell) ); } - pPg->nFree = (pData - pCellptr); + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ pPg->nCell = nCell; pPg->nOverflow = 0; @@ -6019,6 +6019,106 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static u8 *pageFindSlot(MemPage *pPg, int nByte){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + +static int pageInsertArray( + MemPage *pPg, + u8 *pBegin, + u8 **ppData, + u8 *pCellptr, + int nCell, + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + int i; + u8 *aData = pPg->aData; + u8 *pData = *ppData; + for(i=0; iaData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + int nRet = 0; + int i; + u8 *pFree = 0; + int szFree = 0; + + for(i=0; iaData && pCellnFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ static void editPage( MemPage *pPg, /* Edit this page */ int iOld, /* Index of first cell currently on page */ @@ -6027,113 +6127,93 @@ static void editPage( u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ - - if( 1 ){ - u8 * const aData = pPg->aData; - const int hdr = pPg->hdrOffset; - u8 *pBegin = &pPg->aCellIdx[nNew * 2]; - int nFree = pPg->nFree; /* Free bytes on pPg */ - int nCell = pPg->nCell; /* Cells stored on pPg */ - u8 *pData; - u8 *pCellptr; - int i; - int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; #ifdef SQLITE_DEBUG - u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); - memcpy(pTmp, aData, pPg->pBt->usableSize); + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); #endif - /* Remove cells from the start and end of the page */ - if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nShift++; - } - } - nCell -= nShift; - memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); - } - for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nCell--; - } - } - pData = &aData[get2byte(&aData[hdr+5])]; - if( pDataaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + nCell -= pageFreeArray( + pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] + ); + } - /* Add cells to the start of the page */ - if( iNewaCellIdx; - memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); - for(i=iNew; iaCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nAdd, &apCell[iNew], &szCell[iNew] + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + nCell++; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + 1, &apCell[iCell + iNew], &szCell[iCell + iNew] + ) ) goto editpage_fail; } + } - /* Add any overflow cells */ - for(i=0; inOverflow; i++){ - int iCell = (iOld + pPg->aiOvfl[i]) - iNew; - if( iCell>=0 && iCellaCellIdx[iCell * 2]; - int sz = szCell[iCell+iNew]; - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); - pData -= sz; - if( pDataaCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell] + ) ) goto editpage_fail; - /* Append cells to the end of the page */ - pCellptr = &pPg->aCellIdx[nCell*2]; - for(i=iNew+nCell; i<(iNew+nNew); i++){ - pData -= szCell[i]; - if( pDatanCell = nNew; + pPg->nOverflow = 0; - pPg->nFree = nFree; - pPg->nCell = nNew; - pPg->nOverflow = 0; - - put2byte(&aData[hdr+3], pPg->nCell); - put2byte(&aData[hdr+5], pData - aData); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); #ifdef SQLITE_DEBUG - for(i=0; iaCellIdx[i*2]); - if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ - pCell = &pTmp[pCell - aData]; - } - assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } #endif #if 0 - printf("EDIT\n"); +printf("EDIT\n"); #endif - return; - } - + return; editpage_fail: #if 0 printf("REBUILD\n"); @@ -6662,7 +6742,7 @@ static int balance_nonroot( assert( i usableSpace ){ - szNew[k] = subtotal - szCell[i]; + szNew[k] = subtotal - szCell[i] - 2; cntNew[k] = i; if( leafData ){ i--; } subtotal = 0; @@ -7002,6 +7082,7 @@ static int balance_nonroot( editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell); abDone[iPg] = 1; + apNew[iPg]->nFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); assert( apNew[iPg]->nCell==nNewCell ); } From 8e9ba0c335c5dc263f18f42f5c014233c7ad8976 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Oct 2014 17:27:04 +0000 Subject: [PATCH 05/10] Fix some code duplication issues on this branch. Add minor optimizations to the new code. FossilOrigin-Name: 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba --- manifest | 12 +- manifest.uuid | 2 +- src/btree.c | 295 +++++++++++++++++++++----------------------------- 3 files changed, 128 insertions(+), 181 deletions(-) diff --git a/manifest b/manifest index f6d9403e16..347e58fbc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-10-13T18:09:13.737 +C Fix\ssome\scode\sduplication\sissues\son\sthis\sbranch.\sAdd\sminor\soptimizations\sto\sthe\snew\scode. +D 2014-10-14T17:27:04.829 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3d9d1db1d05724346876cf456a850628a9e75331 +F src/btree.c 3c72c5aa96b1a06ea15da24b82c526e6f14010ab F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6594f9b420e2fa642737722ff8521f756ecef227 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 -R 76183a9b7ddeb938a48604dbc4dd8cd6 +P d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 +R ff1ec1f8aba483920a0100e96e95dccf U dan -Z 504cabb1e1fb0c47450f76fec3d8495b +Z 9c750816a0b42cc370356f0c03af45ae diff --git a/manifest.uuid b/manifest.uuid index 32265a861c..065d26bf44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 \ No newline at end of file +58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7136b45b24..acf466304f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1224,6 +1224,57 @@ static int defragmentPage(MemPage *pPage){ return SQLITE_OK; } +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If it does and argument +** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** corruption goes unreported. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + if( pRc ) *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + /* ** Allocate nByte bytes of space from within the B-Tree page passed ** as the first argument. Write into *pIdx the index into pPage->aData[] @@ -1274,33 +1325,12 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+1==top ); testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( data[hdr+7]>=60 ) goto defragment_page; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + int rc = SQLITE_OK; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( rc ) return rc; + if( pSpace ){ + *pIdx = pSpace - data; + return SQLITE_OK; } } @@ -1309,7 +1339,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ -defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -5933,48 +5962,18 @@ static void insertCell( } /* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. */ -static void assemblePage( - MemPage *pPage, /* The page to be assembled */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ -){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ - - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) - && (int)MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); - - pCellptr = pPage->aCellIdx; - cellbody = nUsable; - for(i=0; inFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; -} - - static void rebuildPage( MemPage *pPg, /* Edit this page */ int nCell, /* Final number of cells on page */ @@ -5992,8 +5991,8 @@ static void rebuildPage( i = get2byte(&aData[hdr+5]); memcpy(&pTmp[i], &aData[i], usableSize - i); - pData = &aData[usableSize]; + pData = pEnd; for(i=0; iaData && pCellhdrOffset; - u8 * const aData = pPg->aData; - int iAddr; - int pc; - int usableSize = pPg->pBt->usableSize; - - for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&aData[iAddr], &aData[pc], 2); - aData[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return 0; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&aData[pc+2], x); - } - return &aData[pc + x]; - } - } - - return 0; -} - +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function attempts to +** add the cells stored in the array to page pPg. If it cannot (because +** the page needs to be defragmented before the cells will fit), non-zero +** is returned. Otherwise, if the cells are added successfully, zero is +** returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ static int pageInsertArray( - MemPage *pPg, - u8 *pBegin, - u8 **ppData, - u8 *pCellptr, - int nCell, + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int nCell, /* Number of cells to add to pPg */ u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ int i; u8 *aData = pPg->aData; u8 *pData = *ppData; + const int bFreelist = aData[1] || aData[2]; + assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; inFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. @@ -6206,15 +6205,8 @@ static void editPage( } #endif -#if 0 -printf("EDIT\n"); -#endif - return; editpage_fail: -#if 0 - printf("REBUILD\n"); -#endif /* Unable to edit this page. Rebuild it from scratch instead. */ rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); } @@ -6288,7 +6280,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); + rebuildPage(pNew, 1, &pCell, &szCell); + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map ** with entries for the new page, and any pointer from the @@ -6518,8 +6511,6 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - int aShiftLeft[NB+2]; - int aShiftRight[NB+2]; u8 abDone[NB+2]; Pgno aPgno[NB+2]; u16 aPgFlags[NB+2]; @@ -6887,49 +6878,6 @@ static int balance_nonroot( assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - j = 0; - for(i=0; i %d@%d", apNew[i]->pgno, - pNew->nCell+pNew->nOverflow, j, - cntNew[i] - iFirst, iFirst - ); - for(iCell=iFirst; iCellaData - || apCell[iCell]>=&pNew->aData[pBt->usableSize] - ){ - nCta += szCell[iCell]; - } - } - nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); - nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); - printf(" cta=%d free=%d\n", nCta, nFree); - if( i==(nNew-1) ){ - printf("-----\n"); - fflush(stdout); - } -#endif - - assert( inCell + apNew[i]->nOverflow; - aShiftRight[i] = cntNew[i] - j; - assert( i!=nOld-1 || j==nCell ); - if( j=0 && aShiftLeft[0]==0 ); for(i=0; i=nNew ? i-nNew : nNew-1-i); if( abDone[iPg]==0 - && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) - && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + && (iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1]) + && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) ){ int iNew; int iOld; From e6593d8e8c51fd05c762f34c444b19a74137fedc Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 16:40:49 +0000 Subject: [PATCH 06/10] Fix some minor formatting and code organization issues. FossilOrigin-Name: eab8706dc47aa0a44caf73619de858397c3e0b4e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 16 +++++++--------- src/pager.c | 15 +++++++++++---- src/pager.h | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 396fe366d0..b52c190c13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\swith\sthis\sbranch. -D 2014-10-22T18:42:31.680 +C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. +D 2014-10-24T16:40:49.448 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 37229f3134416f24828af823a3a6f8535923d45b +F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 18be49e363106bdb4c982fa27dbee7a8cd297113 -F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 +F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba 3c933bf95f291f7957580d823dce92c981375a5c -R 738ce949461e4c95ef4d450eb06d976a +P 854a54c6c21e800b0cd999023014813f7c50b23f +R b3aec92126a25950ff876c6cdf2fb4df U dan -Z 1bc59f5d05ced43bcf6fa4690adf0fcc +Z 9e7a15bc5953228881d999c99d9f0682 diff --git a/manifest.uuid b/manifest.uuid index b48e7f9fc1..5987e57b67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -854a54c6c21e800b0cd999023014813f7c50b23f \ No newline at end of file +eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d1457c5f39..b5e8cb2e0c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1233,7 +1233,7 @@ static int defragmentPage(MemPage *pPage){ ** ** This function may detect corruption within pPg. If it does and argument ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. -** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ @@ -6345,7 +6345,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 1 +#if 0 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6522,10 +6522,9 @@ static int balance_nonroot( u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - - u8 abDone[NB+2]; - Pgno aPgno[NB+2]; - u16 aPgFlags[NB+2]; + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; @@ -6868,8 +6867,7 @@ static int balance_nonroot( } } if( apNew[i]->pgno!=iMin ){ - apNew[i]->pDbPage->flags = flags; - sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } } @@ -7090,7 +7088,7 @@ static int balance_nonroot( freePage(apOld[i], &rc); } -#if 1 +#if 0 if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while diff --git a/src/pager.c b/src/pager.c index 74667afab0..dc79e6754a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6845,16 +6845,23 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } +#endif -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ PgHdr *pPg = (PgHdr*)pPage; - assert( pPg->flags & PGHDR_DIRTY ); + assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); assert( !subjRequiresPage(pPg) ); + assert( pPg->pgno!=iNew ); + pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); } -#endif - /* ** Return a pointer to the data for the specified page. */ diff --git a/src/pager.h b/src/pager.h index f3a04bbca6..764c7bd197 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,7 +188,7 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); -void sqlite3PagerRekey(DbPage*, Pgno); +void sqlite3PagerRekey(DbPage*, Pgno, u16); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); From 23eba45d23291617e5f69ffb15c0cfbb1bab5121 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 18:43:57 +0000 Subject: [PATCH 07/10] Fix some issues in the new code on this branch related to the handling of corrupt databases. FossilOrigin-Name: 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 14 ++++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b52c190c13..10be86a2cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. -D 2014-10-24T16:40:49.448 +C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. +D 2014-10-24T18:43:57.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 +F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 854a54c6c21e800b0cd999023014813f7c50b23f -R b3aec92126a25950ff876c6cdf2fb4df +P eab8706dc47aa0a44caf73619de858397c3e0b4e +R 8a34c36d0ed8bbbc0bb7360d0dc30d47 U dan -Z 9e7a15bc5953228881d999c99d9f0682 +Z 610a2814ec6cdf214c41e43dd2355678 diff --git a/manifest.uuid b/manifest.uuid index 5987e57b67..11462e357c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file +19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b5e8cb2e0c..f326f713fb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1386,7 +1386,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ assert( iStart<=iLast ); @@ -6065,7 +6065,7 @@ static int pageInsertArray( u8 *aData = pPg->aData; u8 *pData = *ppData; const int bFreelist = aData[1] || aData[2]; - assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; iaCellIdx[i*2]); if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ @@ -6866,7 +6866,13 @@ static int balance_nonroot( flags = aPgFlags[j]; } } - if( apNew[i]->pgno!=iMin ){ + if( iMin==0 ){ + /* This case can only occur if aPgno[] contains duplicate page + ** numbers. Which can only happen if the database is corrupt. */ + assert( CORRUPT_DB ); + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + }else if( apNew[i]->pgno!=iMin ){ sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } From 31f4e99d449b5938f46723f6d2ec96595ff88513 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 20:57:03 +0000 Subject: [PATCH 08/10] Ensure that the "Any prior cache entry associated with newKey is guaranteed not to be pinned" guarantee made to xRekey implementations is not violated. FossilOrigin-Name: ecc3544e712041736af7c7b4f34864a1f2e30ff7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 30 ++++++++++++------------------ src/pager.c | 5 +---- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 10be86a2cc..6c0464217b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. -D 2014-10-24T18:43:57.313 +C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. +D 2014-10-24T20:57:03.500 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea +F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eab8706dc47aa0a44caf73619de858397c3e0b4e -R 8a34c36d0ed8bbbc0bb7360d0dc30d47 +P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc +R c9d1872515b77329bc9fbfe4c8e91668 U dan -Z 610a2814ec6cdf214c41e43dd2355678 +Z 931e363227b1144b3ac19bee87a82036 diff --git a/manifest.uuid b/manifest.uuid index 11462e357c..470b2345ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file +ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f326f713fb..83ed66414c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6856,25 +6856,19 @@ static int balance_nonroot( aPgFlags[i] = apNew[i]->pDbPage->flags; } for(i=0; ipgno); - Pgno iMin = 0; - u16 flags = 0; - for(j=0; jiGt && (iMin==0 || iPgnopgno!=iMin ){ - sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); - apNew[i]->pgno = iMin; + pgno = aPgno[iBest]; + aPgno[iBest] = 0xffffffff; + if( iBest!=i ){ + if( iBest>i ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; } } diff --git a/src/pager.c b/src/pager.c index dc79e6754a..997f842d0a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6853,10 +6853,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** page number to iNew and sets the value of the PgHdr.flags field to ** the value passed as the third parameter. */ -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ - PgHdr *pPg = (PgHdr*)pPage; - assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); - assert( !subjRequiresPage(pPg) ); +void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ assert( pPg->pgno!=iNew ); pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); From 89ca0b3878bf964c202e1d0c984b39a9eefaafb0 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Oct 2014 20:36:28 +0000 Subject: [PATCH 09/10] Further modifications to new code to better handle corrupt databases. FossilOrigin-Name: 1a8cf0a043347772ac54d150d634c32845beee8b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 30 +++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6c0464217b..1a5be4bf57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. -D 2014-10-24T20:57:03.500 +C Further\smodifications\sto\snew\scode\sto\sbetter\shandle\scorrupt\sdatabases. +D 2014-10-25T20:36:28.557 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 +F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc -R c9d1872515b77329bc9fbfe4c8e91668 +P ecc3544e712041736af7c7b4f34864a1f2e30ff7 +R 17dc589efbb3f16f6448c9d9d1ae2e24 U dan -Z 931e363227b1144b3ac19bee87a82036 +Z c615066e82cbd286f5597ce396069fd7 diff --git a/manifest.uuid b/manifest.uuid index 470b2345ea..23643921ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file +1a8cf0a043347772ac54d150d634c32845beee8b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 83ed66414c..46ce39abc7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6099,6 +6099,7 @@ static int pageFreeArray( ){ u8 * const aData = pPg->aData; u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; int i; u8 *pFree = 0; @@ -6106,12 +6107,13 @@ static int pageFreeArray( for(i=0; iaData && pCell=pStart && pCellpEnd ) return 0; }else{ pFree = pCell; szFree += sz; @@ -6854,6 +6856,18 @@ static int balance_nonroot( for(i=0; ipgno; aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; jnFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); + rc = defragmentPage(apNew[0]); + if( rc==SQLITE_OK ){ + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + } }else if( ISAUTOVACUUM && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken From 61e94c9deb9d0edff1fdaa443fe7668cbfacedb4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Oct 2014 08:02:16 +0000 Subject: [PATCH 10/10] If a free-slot is found within a page, but using that free-slot would fragment the page further and there are already at least 60 fragmented bytes, degragment the page. This matches the behaviour of the trunk. FossilOrigin-Name: 1f80f8c136ac970dcc7fb2337263dc5922e348c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bafcf0fb91..a088218356 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\swith\sthis\sbranch. -D 2014-10-27T07:01:04.796 +C If\sa\sfree-slot\sis\sfound\swithin\sa\spage,\sbut\susing\sthat\sfree-slot\swould\sfragment\sthe\spage\sfurther\sand\sthere\sare\salready\sat\sleast\s60\sfragmented\sbytes,\sdegragment\sthe\spage.\sThis\smatches\sthe\sbehaviour\sof\sthe\strunk. +D 2014-10-27T08:02:16.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 +F src/btree.c 9790fb4df51d36861bcbb8cd0a9b41586cbae699 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a8cf0a043347772ac54d150d634c32845beee8b 0f08924fe0c52a85a103f67bee9809e0f8f884b0 -R 9858b90cbf912e211c65716d59c94e8a +P a13df3013bbac4a0d4fce5cef1376c857508c1c5 +R 05f44d14ab419727640258148977021c U dan -Z daf2b363c619dfc32da3ad287a4e25ec +Z 95c7c43e9818bc003f29ce4926da458f diff --git a/manifest.uuid b/manifest.uuid index 962b2e06bb..92680eaebb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a13df3013bbac4a0d4fce5cef1376c857508c1c5 \ No newline at end of file +1f80f8c136ac970dcc7fb2337263dc5922e348c3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 46ce39abc7..9b14dc2b6e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1235,8 +1235,12 @@ static int defragmentPage(MemPage *pPage){ ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. +** +** If a slot of at least nByte bytes is found but cannot be used because +** there are already at least 60 fragmented bytes on the page, return NULL. +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true. */ -static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ const int hdr = pPg->hdrOffset; u8 * const aData = pPg->aData; int iAddr; @@ -1255,7 +1259,10 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ testcase( x==4 ); testcase( x==3 ); if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; + if( aData[hdr+7]>=60 ){ + if( pbDefrag ) *pbDefrag = 1; + return 0; + } /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); @@ -1326,8 +1333,10 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ int rc = SQLITE_OK; - u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + int bDefrag = 0; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); if( rc ) return rc; + if( bDefrag ) goto defragment_page; if( pSpace ){ *pIdx = pSpace - data; return SQLITE_OK; @@ -1339,6 +1348,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ + defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -6069,7 +6079,7 @@ static int pageInsertArray( for(i=0; i