paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge the experimental changes to use a merge-sort when creating an index into the trunk.

Browse Source 
FossilOrigin-Name: 346a453d1f1d615f1e9e7923a27dad84b8727fb1
This commit is contained in:
dan
2011-08-12 16:47:12 +00:00
parent 90315a2417 a9f39fde23
commit 7b14e59bfd
19 changed files with 3380 additions and 4340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.in
+6 -2
View File
@@ -177,8 +177,8 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
table.lo tokenize.lo trigger.lo \
update.lo util.lo vacuum.lo \
vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbetrace.lo \
wal.lo walker.lo where.lo utf.lo vtab.lo
vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo
# Object files for the amalgamation.
#
@@ -274,6 +274,7 @@ SRC = \
$(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbeblob.c \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbesort.c \
$(TOP)/src/vdbetrace.c \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vtab.c \
@@ -740,6 +741,9 @@ vdbeblob.lo: $(TOP)/src/vdbeblob.c $(HDR)
vdbemem.lo: $(TOP)/src/vdbemem.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbemem.c
vdbesort.lo: $(TOP)/src/vdbesort.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbesort.c
vdbetrace.lo: $(TOP)/src/vdbetrace.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbetrace.c
configure
Vendored
+2051 -4296
View File
File diff suppressed because it is too large Load Diff
main.mk
+3 -2
View File
@@ -65,8 +65,8 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \
random.o resolve.o rowset.o rtree.o select.o status.o \
table.o tokenize.o trigger.o \
update.o util.o vacuum.o \
vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbetrace.o \
wal.o walker.o where.o utf.o vtab.o
vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
vdbetrace.o wal.o walker.o where.o utf.o vtab.o
@@ -155,6 +155,7 @@ SRC = \
$(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbeblob.c \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbesort.c \
$(TOP)/src/vdbetrace.c \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vtab.c \
manifest
+23 -20
View File
@@ -1,7 +1,7 @@
C Make\sthe\sopenDirectory\sroutine\sin\sos_unix.c\soverrideable\sso\sthat\sit\scan\nbe\sturned\sinto\sa\sharmless\sno-op\sfor\sthe\schromium\ssandbox.
D 2011-08-10T01:52:12.736
C Merge\sthe\sexperimental\schanges\sto\suse\sa\smerge-sort\swhen\screating\san\sindex\sinto\sthe\strunk.
D 2011-08-12T16:47:12.815
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc
F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F Makefile.msc 9fd0401b0134b565a9c7d8ed49dc044cc89f2096
F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151
@@ -23,7 +23,7 @@ F art/src_logo.gif 9341ef09f0e53cd44c0c9b6fc3c16f7f3d6c2ad9
F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
F config.h.in 405a958bdb3af382a809dccb08a44694923ddd61
F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
F configure 93e7e695581fa7bef4949161453d9845c5592ad0 x
F configure 4010a2e577b7d43c429672858d442ac5a570fdcd x
F configure.ac 298a759c086e72c013da459c2aec02a104f4224f
F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
F doc/lemon.html 3091574143dd3415669b6745843ff8d011d33549
@@ -104,7 +104,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F main.mk 50adb7368bd0faa888eb4634b874d542d44f3de3
F main.mk a812efc287a904c3e04098f090de036b264ba779
F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
@@ -124,13 +124,13 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c
F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f
F src/btree.c a30bdcc27eedc36a38a3a11e1ba83de9a6729f7e
F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce
F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3
F src/build.c 19a8957a442d922a0d6ed1a5dd67b63202fc3260
F src/build.c 77b40abf79e1541356f552c9681b0b431aba672b
F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a
F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4
F src/ctime.c 7f283795650dd4122cc07bd02193c40193b32cc6
F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3
F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8
F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07
@@ -201,7 +201,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad
F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e
F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de
F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2
F src/test_config.c b4648b103586d2ae863056080c657680f6fa4825
F src/test_config.c 9bc44df77f22cd0648c651fcd459353b8a984d7b
F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094
F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
F src/test_func.c cbdec5cededa0761daedde5baf06004a9bf416b5
@@ -228,7 +228,7 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd
F src/test_syscall.c 162c4ec0137a549c009bb9ecab550527743cfc5d
F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa
F src/test_thread.c fe9a7803fc1d69cccb60f016f28c1cedf2d9fcfa
F src/test_vfs.c 956cb3f5cbd2a0d09129540e615bb0fb761c083d
F src/test_vfs.c b0baec983bd6f872715a4b44c8f39104fec333af
F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d
F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290
F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
@@ -238,13 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec
F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0
F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7
F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e
F src/vdbe.c 49d834f0fe49d305e07f9c212e94007fda2028e9
F src/vdbe.c 22d0d5bc8e8e467e1e0fa20a0a6b21c297f5601f
F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1
F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45
F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096
F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98
F src/vdbeaux.c 4d100407e3c72e163854aff8903d19d5ecdf46c0
F src/vdbeaux.c 8fb978eb73a97b34d352dd3ef3bff35b1b3fa7e9
F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3
F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b
F src/vdbesort.c 3e6b2268326ee075ae5e8cece4e469e5d1d4a859
F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114
F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582
F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9
@@ -510,7 +511,9 @@ F test/incrvacuum_ioerr.test 22f208d01c528403240e05beecc41dc98ed01637
F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097
F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6
F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7
F test/index4.test c82a59c9ae2ac01804bdb100162dca057318f40f
F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad
F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
F test/insert.test aef273dd1cee84cc92407469e6bd1b3cdcb76908
F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435
@@ -616,7 +619,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
F test/pcache2.test 9f9357bb0f463b87bdf695646024ed2031a0c85a
F test/permutations.test 2c845b5c8cd2abb32f07076734d10e6a9fd19f26
F test/permutations.test ad17319066a90e2db71823c3ff104795ffc71b31
F test/pragma.test c8108e01da04f16e67e5754e610bc62c1b993f6c
F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947
F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
@@ -880,7 +883,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
F test/wal.test e11da8d5ea8a38a247339455098357e9adf63d76
F test/wal2.test a2caa432d056596c6ee815d2743b2ced86f67a59
F test/wal2.test ad6412596815f553cd30f271d291ab003092bc7e
F test/wal3.test 18da4e65c30c43c646ad40e145e9a074e4062fc9
F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
F test/wal5.test 08e145a352b1223930c7f0a1de82a8747a99c322
@@ -928,7 +931,7 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309
F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
F tool/mksqlite3c.tcl 1fa0ed9cfdc768bf5de7e65fda8d97a46dd2a7e6
F tool/mksqlite3c.tcl d8c0d3065bc23fd9e27d59bfebd34df203fe6b08
F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8
F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87
F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091
@@ -957,7 +960,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5
F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262
P dfa22ed4387f9526b74d5265503c7c8e9d559903
R bf22196e9aa98f18724e5d2624f7dcbf
U drh
Z 2ec35109792cc0dc5d4f7cebd4d85034
P 6b236069e1ea3c99ff0a007a790d4baebda70b13 69e4b41de22968c565c6575707a73956e3b20ff4
R fced7a7577e38a6f428a347e92769773
U dan
Z 3bbbe4efed9782cd51a19bb264baec63
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
6b236069e1ea3c99ff0a007a790d4baebda70b13
346a453d1f1d615f1e9e7923a27dad84b8727fb1
src/btree.c
+10 -1
View File
@@ -7277,9 +7277,16 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){
return rc;
}
int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){
BtShared *pBt = p->pBt;
int rc;
sqlite3BtreeEnter(p);
rc = btreeDropTable(p, iTable, piMoved);
if( (pBt->openFlags&BTREE_SINGLE) ){
pBt->nPage = 0;
sqlite3PagerTruncateImage(pBt->pPager, 1);
rc = newDatabase(pBt);
}else{
rc = btreeDropTable(p, iTable, piMoved);
}
sqlite3BtreeLeave(p);
return rc;
}
@@ -8168,3 +8175,5 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
pBt->doNotUseWAL = 0;
return rc;
}
src/build.c
+32 -2
View File
@@ -2324,6 +2324,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
Table *pTab = pIndex->pTable; /* The table that is indexed */
int iTab = pParse->nTab++; /* Btree cursor used for pTab */
int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */
int iSorter = iTab; /* Cursor opened by OpenSorter (if in use) */
int addr1; /* Address of top of loop */
int tnum; /* Root page of index */
Vdbe *v; /* Generate code into this virtual machine */
@@ -2333,6 +2334,15 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
sqlite3 *db = pParse->db; /* The database connection */
int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema);
/* Set bUseSorter to use OP_OpenSorter, or clear it to insert directly
** into the index. The sorter is used unless either OMIT_MERGE_SORT is
** defined or the system is configured to store temp files in-memory. */
#ifdef SQLITE_OMIT_MERGE_SORT
static const int bUseSorter = 0;
#else
const int bUseSorter = !sqlite3TempInMemory(pParse->db);
#endif
#ifndef SQLITE_OMIT_AUTHORIZATION
if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0,
db->aDb[iDb].zName ) ){
@@ -2357,10 +2367,28 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
if( memRootPage>=0 ){
sqlite3VdbeChangeP5(v, 1);
}
/* Open the sorter cursor if we are to use one. */
if( bUseSorter ){
iSorter = pParse->nTab++;
sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO);
}
/* Open the table. Loop through all rows of the table, inserting index
** records into the sorter. */
sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead);
addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0);
regRecord = sqlite3GetTempReg(pParse);
regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1);
if( bUseSorter ){
sqlite3VdbeAddOp2(v, OP_IdxInsert, iSorter, regRecord);
sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1);
sqlite3VdbeJumpHere(v, addr1);
addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSorter, 0);
sqlite3VdbeAddOp2(v, OP_RowKey, iSorter, regRecord);
}
if( pIndex->onError!=OE_None ){
const int regRowid = regIdxKey + pIndex->nColumn;
const int j2 = sqlite3VdbeCurrentAddr(v) + 2;
@@ -2379,13 +2407,15 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
sqlite3HaltConstraint(
pParse, OE_Abort, "indexed columns are not unique", P4_STATIC);
}
sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord);
sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter);
sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
sqlite3ReleaseTempReg(pParse, regRecord);
sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1);
sqlite3VdbeAddOp2(v, OP_Next, iSorter, addr1+1);
sqlite3VdbeJumpHere(v, addr1);
sqlite3VdbeAddOp1(v, OP_Close, iTab);
sqlite3VdbeAddOp1(v, OP_Close, iIdx);
sqlite3VdbeAddOp1(v, OP_Close, iSorter);
}
/*
src/ctime.c
+3
View File
@@ -257,6 +257,9 @@ static const char * const azCompileOpt[] = {
#ifdef SQLITE_OMIT_MEMORYDB
"OMIT_MEMORYDB",
#endif
#ifdef SQLITE_OMIT_MERGE_SORT
"OMIT_MERGE_SORT",
#endif
#ifdef SQLITE_OMIT_OR_OPTIMIZATION
"OMIT_OR_OPTIMIZATION",
#endif
src/test_config.c
+6
View File
@@ -363,6 +363,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double",
Tcl_SetVar2(interp, "sqlite_options", "memorymanage", "0", TCL_GLOBAL_ONLY);
#endif
#ifdef SQLITE_OMIT_MERGE_SORT
Tcl_SetVar2(interp, "sqlite_options", "mergesort", "0", TCL_GLOBAL_ONLY);
#else
Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
#endif
#ifdef SQLITE_OMIT_OR_OPTIMIZATION
Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY);
#else
src/test_vfs.c
+19 -2
View File
@@ -123,6 +123,8 @@ struct Testvfs {
#define TESTVFS_TRUNCATE_MASK 0x00002000
#define TESTVFS_ACCESS_MASK 0x00004000
#define TESTVFS_FULLPATHNAME_MASK 0x00008000
#define TESTVFS_READ_MASK 0x00010000
#define TESTVFS_ALL_MASK 0x0001FFFF
@@ -325,8 +327,22 @@ static int tvfsRead(
int iAmt,
sqlite_int64 iOfst
){
TestvfsFd *p = tvfsGetFd(pFile);
return sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst);
int rc = SQLITE_OK;
TestvfsFd *pFd = tvfsGetFd(pFile);
Testvfs *p = (Testvfs *)pFd->pVfs->pAppData;
if( p->pScript && p->mask&TESTVFS_READ_MASK ){
tvfsExecTcl(p, "xRead",
Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0
);
tvfsResultCode(p, &rc);
}
if( rc==SQLITE_OK && p->mask&TESTVFS_READ_MASK && tvfsInjectIoerr(p) ){
rc = SQLITE_IOERR;
}
if( rc==SQLITE_OK ){
rc = sqlite3OsRead(pFd->pReal, zBuf, iAmt, iOfst);
}
return rc;
}
/*
@@ -1030,6 +1046,7 @@ static int testvfs_obj_cmd(
{ "xSync", TESTVFS_SYNC_MASK },
{ "xDelete", TESTVFS_DELETE_MASK },
{ "xWrite", TESTVFS_WRITE_MASK },
{ "xRead", TESTVFS_READ_MASK },
{ "xTruncate", TESTVFS_TRUNCATE_MASK },
{ "xOpen", TESTVFS_OPEN_MASK },
{ "xClose", TESTVFS_CLOSE_MASK },
src/vdbe.c
+43 -13
View File
@@ -157,6 +157,13 @@ int sqlite3_found_count = 0;
*/
#define ExpandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0)
/* Return true if the cursor was opened using the OP_OpenSorter opcode. */
#ifdef SQLITE_OMIT_MERGE_SORT
# define isSorter(x) 0
#else
# define isSorter(x) ((x)->pSorter!=0)
#endif
/*
** Argument pMem points at a register that will be passed to a
** user-defined function or returned to the user as the result of a query.
@@ -3149,6 +3156,7 @@ case OP_OpenWrite: {
** by this opcode will be used for automatically created transient
** indices in joins.
*/
case OP_OpenSorter:
case OP_OpenAutoindex:
case OP_OpenEphemeral: {
VdbeCursor *pCx;
@@ -3193,6 +3201,11 @@ case OP_OpenEphemeral: {
}
pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED);
pCx->isIndex = !pCx->isTable;
#ifndef SQLITE_OMIT_MERGE_SORT
if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){
rc = sqlite3VdbeSorterInit(db, pCx);
}
#endif
break;
}
@@ -4082,6 +4095,13 @@ case OP_RowData: {
assert( pC!=0 );
assert( pC->nullRow==0 );
assert( pC->pseudoTableReg==0 );
if( isSorter(pC) ){
assert( pOp->opcode==OP_RowKey );
rc = sqlite3VdbeSorterRowkey(pC, pOut);
break;
}
assert( pC->pCursor!=0 );
pCrsr = pC->pCursor;
assert( sqlite3BtreeCursorIsValid(pCrsr) );
@@ -4262,7 +4282,9 @@ case OP_Rewind: { /* jump */
pC = p->apCsr[pOp->p1];
assert( pC!=0 );
res = 1;
if( (pCrsr = pC->pCursor)!=0 ){
if( isSorter(pC) ){
rc = sqlite3VdbeSorterRewind(db, pC, &res);
}else if( (pCrsr = pC->pCursor)!=0 ){
rc = sqlite3BtreeFirst(pCrsr, &res);
pC->atFirst = res==0 ?1:0;
pC->deferredMoveto = 0;
@@ -4316,15 +4338,20 @@ case OP_Next: { /* jump */
if( pC==0 ){
break; /* See ticket #2273 */
}
pCrsr = pC->pCursor;
if( pCrsr==0 ){
pC->nullRow = 1;
break;
if( isSorter(pC) ){
assert( pOp->opcode==OP_Next );
rc = sqlite3VdbeSorterNext(db, pC, &res);
}else{
pCrsr = pC->pCursor;
if( pCrsr==0 ){
pC->nullRow = 1;
break;
}
res = 1;
assert( pC->deferredMoveto==0 );
rc = pOp->opcode==OP_Next ? sqlite3BtreeNext(pCrsr, &res) :
sqlite3BtreePrevious(pCrsr, &res);
}
res = 1;
assert( pC->deferredMoveto==0 );
rc = pOp->opcode==OP_Next ? sqlite3BtreeNext(pCrsr, &res) :
sqlite3BtreePrevious(pCrsr, &res);
pC->nullRow = (u8)res;
pC->cacheStatus = CACHE_STALE;
if( res==0 ){
@@ -4368,10 +4395,13 @@ case OP_IdxInsert: { /* in2 */
if( rc==SQLITE_OK ){
nKey = pIn2->n;
zKey = pIn2->z;
rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3,
((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
);
assert( pC->deferredMoveto==0 );
rc = sqlite3VdbeSorterWrite(db, pC, nKey);
if( rc==SQLITE_OK ){
rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3,
((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
);
assert( pC->deferredMoveto==0 );
}
pC->cacheStatus = CACHE_STALE;
}
}
src/vdbeInt.h
+20
View File
@@ -30,6 +30,9 @@ typedef struct VdbeOp Op;
*/
typedef unsigned char Bool;
/* Opaque type used by code in vdbesort.c */
typedef struct VdbeSorter VdbeSorter;
/*
** A cursor is a pointer into a single BTree within a database file.
** The cursor can seek to a BTree entry with a particular key, or
@@ -61,6 +64,7 @@ struct VdbeCursor {
i64 seqCount; /* Sequence counter */
i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */
i64 lastRowid; /* Last rowid from a Next or NextIdx operation */
VdbeSorter *pSorter; /* Sorter object for OP_OpenSorter cursors */
/* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or
** OP_IsUnique opcode on this cursor. */
@@ -388,6 +392,22 @@ void sqlite3VdbeFrameDelete(VdbeFrame*);
int sqlite3VdbeFrameRestore(VdbeFrame *);
void sqlite3VdbeMemStoreType(Mem *pMem);
#ifdef SQLITE_OMIT_MERGE_SORT
# define sqlite3VdbeSorterInit(Y,Z) SQLITE_OK
# define sqlite3VdbeSorterWrite(X,Y,Z) SQLITE_OK
# define sqlite3VdbeSorterClose(Y,Z)
# define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK
# define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK
# define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK
#else
int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *);
int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, int);
void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *);
int sqlite3VdbeSorterRowkey(VdbeCursor *, Mem *);
int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *);
int sqlite3VdbeSorterNext(sqlite3 *, VdbeCursor *, int *);
#endif
#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0
void sqlite3VdbeEnter(Vdbe*);
void sqlite3VdbeLeave(Vdbe*);
src/vdbeaux.c
+1
View File
@@ -1565,6 +1565,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){
if( pCx==0 ){
return;
}
sqlite3VdbeSorterClose(p->db, pCx);
if( pCx->pBt ){
sqlite3BtreeClose(pCx->pBt);
/* The pCx->pCursor will be close automatically, if it exists, by
src/vdbesort.c
+705
View File
@@ -0,0 +1,705 @@
/*
** 2011 July 9
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains code for the VdbeSorter object, used in concert with
** a VdbeCursor to sort large numbers of keys (as may be required, for
** example, by CREATE INDEX statements on tables too large to fit in main
** memory).
*/
#include "sqliteInt.h"
#include "vdbeInt.h"
#ifndef SQLITE_OMIT_MERGE_SORT
typedef struct VdbeSorterIter VdbeSorterIter;
/*
** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES:
**
** As keys are added to the sorter, they are written to disk in a series
** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly
** the same as the cache-size allowed for temporary databases. In order
** to allow the caller to extract keys from the sorter in sorted order,
** all PMAs currently stored on disk must be merged together. This comment
** describes the data structure used to do so. The structure supports
** merging any number of arrays in a single pass with no redundant comparison
** operations.
**
** The aIter[] array contains an iterator for each of the PMAs being merged.
** An aIter[] iterator either points to a valid key or else is at EOF. For
** the purposes of the paragraphs below, we assume that the array is actually
** N elements in size, where N is the smallest power of 2 greater to or equal
** to the number of iterators being merged. The extra aIter[] elements are
** treated as if they are empty (always at EOF).
**
** The aTree[] array is also N elements in size. The value of N is stored in
** the VdbeSorter.nTree variable.
**
** The final (N/2) elements of aTree[] contain the results of comparing
** pairs of iterator keys together. Element i contains the result of
** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the
** aTree element is set to the index of it.
**
** For the purposes of this comparison, EOF is considered greater than any
** other key value. If the keys are equal (only possible with two EOF
** values), it doesn't matter which index is stored.
**
** The (N/4) elements of aTree[] that preceed the final (N/2) described
** above contains the index of the smallest of each block of 4 iterators.
** And so on. So that aTree[1] contains the index of the iterator that
** currently points to the smallest key value. aTree[0] is unused.
**
** Example:
**
** aIter[0] -> Banana
** aIter[1] -> Feijoa
** aIter[2] -> Elderberry
** aIter[3] -> Currant
** aIter[4] -> Grapefruit
** aIter[5] -> Apple
** aIter[6] -> Durian
** aIter[7] -> EOF
**
** aTree[] = { X, 5 0, 5 0, 3, 5, 6 }
**
** The current element is "Apple" (the value of the key indicated by
** iterator 5). When the Next() operation is invoked, iterator 5 will
** be advanced to the next key in its segment. Say the next key is
** "Eggplant":
**
** aIter[5] -> Eggplant
**
** The contents of aTree[] are updated first by comparing the new iterator
** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator
** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree.
** The value of iterator 6 - "Durian" - is now smaller than that of iterator
** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (Banana<Durian),
** so the value written into element 1 of the array is 0. As follows:
**
** aTree[] = { X, 0 0, 6 0, 3, 5, 6 }
**
** In other words, each time we advance to the next sorter element, log2(N)
** key comparison operations are required, where N is the number of segments
** being merged (rounded up to the next power of 2).
*/
struct VdbeSorter {
int nWorking; /* Start a new b-tree after this many pages */
int nBtree; /* Current size of b-tree contents as PMA */
int nTree; /* Used size of aTree/aIter (power of 2) */
VdbeSorterIter *aIter; /* Array of iterators to merge */
int *aTree; /* Current state of incremental merge */
i64 iWriteOff; /* Current write offset within file pTemp1 */
i64 iReadOff; /* Current read offset within file pTemp1 */
sqlite3_file *pTemp1; /* PMA file 1 */
int nPMA; /* Number of PMAs stored in pTemp1 */
};
/*
** The following type is an iterator for a PMA. It caches the current key in
** variables nKey/aKey. If the iterator is at EOF, pFile==0.
*/
struct VdbeSorterIter {
i64 iReadOff; /* Current read offset */
i64 iEof; /* 1 byte past EOF for this iterator */
sqlite3_file *pFile; /* File iterator is reading from */
int nAlloc; /* Bytes of space at aAlloc */
u8 *aAlloc; /* Allocated space */
int nKey; /* Number of bytes in key */
u8 *aKey; /* Pointer to current key */
};
/* Minimum allowable value for the VdbeSorter.nWorking variable */
#define SORTER_MIN_WORKING 10
/* Maximum number of segments to merge in a single pass. */
#define SORTER_MAX_MERGE_COUNT 16
/*
** Free all memory belonging to the VdbeSorterIter object passed as the second
** argument. All structure fields are set to zero before returning.
*/
static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){
sqlite3DbFree(db, pIter->aAlloc);
memset(pIter, 0, sizeof(VdbeSorterIter));
}
/*
** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if
** no error occurs, or an SQLite error code if one does.
*/
static int vdbeSorterIterNext(
sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */
VdbeSorterIter *pIter /* Iterator to advance */
){
int rc; /* Return Code */
int nRead; /* Number of bytes read */
int nRec; /* Size of record in bytes */
int iOff; /* Size of serialized size varint in bytes */
nRead = pIter->iEof - pIter->iReadOff;
if( nRead>5 ) nRead = 5;
if( nRead<=0 ){
/* This is an EOF condition */
vdbeSorterIterZero(db, pIter);
return SQLITE_OK;
}
rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff);
iOff = getVarint32(pIter->aAlloc, nRec);
if( rc==SQLITE_OK && (iOff+nRec)>nRead ){
int nRead2; /* Number of extra bytes to read */
if( (iOff+nRec)>pIter->nAlloc ){
int nNew = pIter->nAlloc*2;
while( (iOff+nRec)>nNew ) nNew = nNew*2;
pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew);
if( !pIter->aAlloc ) return SQLITE_NOMEM;
pIter->nAlloc = nNew;
}
nRead2 = iOff + nRec - nRead;
rc = sqlite3OsRead(
pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead
);
}
assert( nRec>0 || rc!=SQLITE_OK );
pIter->iReadOff += iOff+nRec;
pIter->nKey = nRec;
pIter->aKey = &pIter->aAlloc[iOff];
return rc;
}
/*
** Write a single varint, value iVal, to file-descriptor pFile. Return
** SQLITE_OK if successful, or an SQLite error code if some error occurs.
**
** The value of *piOffset when this function is called is used as the byte
** offset in file pFile to write to. Before returning, *piOffset is
** incremented by the number of bytes written.
*/
static int vdbeSorterWriteVarint(
sqlite3_file *pFile, /* File to write to */
i64 iVal, /* Value to write as a varint */
i64 *piOffset /* IN/OUT: Write offset in file pFile */
){
u8 aVarint[9]; /* Buffer large enough for a varint */
int nVarint; /* Number of used bytes in varint */
int rc; /* Result of write() call */
nVarint = sqlite3PutVarint(aVarint, iVal);
rc = sqlite3OsWrite(pFile, aVarint, nVarint, *piOffset);
*piOffset += nVarint;
return rc;
}
/*
** Read a single varint from file-descriptor pFile. Return SQLITE_OK if
** successful, or an SQLite error code if some error occurs.
**
** The value of *piOffset when this function is called is used as the
** byte offset in file pFile from whence to read the varint. If successful
** (i.e. if no IO error occurs), then *piOffset is set to the offset of
** the first byte past the end of the varint before returning. *piVal is
** set to the integer value read. If an error occurs, the final values of
** both *piOffset and *piVal are undefined.
*/
static int vdbeSorterReadVarint(
sqlite3_file *pFile, /* File to read from */
i64 iEof, /* Total number of bytes in file */
i64 *piOffset, /* IN/OUT: Read offset in pFile */
i64 *piVal /* OUT: Value read from file */
){
u8 aVarint[9]; /* Buffer large enough for a varint */
i64 iOff = *piOffset; /* Offset in file to read from */
int nRead = 9; /* Number of bytes to read from file */
int rc; /* Return code */
assert( iEof>iOff );
if( (iEof-iOff)<nRead ){
nRead = iEof-iOff;
}
rc = sqlite3OsRead(pFile, aVarint, nRead, iOff);
if( rc==SQLITE_OK ){
*piOffset += getVarint(aVarint, (u64 *)piVal);
}
return rc;
}
/*
** Initialize iterator pIter to scan through the PMA stored in file pFile
** starting at offset iStart and ending at offset iEof-1. This function
** leaves the iterator pointing to the first key in the PMA (or EOF if the
** PMA is empty).
*/
static int vdbeSorterIterInit(
sqlite3 *db, /* Database handle */
VdbeSorter *pSorter, /* Sorter object */
i64 iStart, /* Start offset in pFile */
VdbeSorterIter *pIter, /* Iterator to populate */
i64 *pnByte /* IN/OUT: Increment this value by PMA size */
){
int rc;
assert( pSorter->iWriteOff>iStart );
assert( pIter->aAlloc==0 );
pIter->pFile = pSorter->pTemp1;
pIter->iReadOff = iStart;
pIter->nAlloc = 128;
pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc);
if( !pIter->aAlloc ){
rc = SQLITE_NOMEM;
}else{
i64 iEof = pSorter->iWriteOff; /* EOF of file pSorter->pTemp1 */
i64 nByte; /* Total size of PMA in bytes */
rc = vdbeSorterReadVarint(pSorter->pTemp1, iEof, &pIter->iReadOff, &nByte);
*pnByte += nByte;
pIter->iEof = pIter->iReadOff + nByte;
}
if( rc==SQLITE_OK ){
rc = vdbeSorterIterNext(db, pIter);
}
return rc;
}
/*
** This function is called to compare two iterator keys when merging
** multiple b-tree segments. Parameter iOut is the index of the aTree[]
** value to recalculate.
*/
static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){
VdbeSorter *pSorter = pCsr->pSorter;
int i1;
int i2;
int iRes;
VdbeSorterIter *p1;
VdbeSorterIter *p2;
assert( iOut<pSorter->nTree && iOut>0 );
if( iOut>=(pSorter->nTree/2) ){
i1 = (iOut - pSorter->nTree/2) * 2;
i2 = i1 + 1;
}else{
i1 = pSorter->aTree[iOut*2];
i2 = pSorter->aTree[iOut*2+1];
}
p1 = &pSorter->aIter[i1];
p2 = &pSorter->aIter[i2];
if( p1->pFile==0 ){
iRes = i2;
}else if( p2->pFile==0 ){
iRes = i1;
}else{
char aSpace[150];
UnpackedRecord *r1;
r1 = sqlite3VdbeRecordUnpack(
pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace)
);
if( r1==0 ) return SQLITE_NOMEM;
if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){
iRes = i1;
}else{
iRes = i2;
}
sqlite3VdbeDeleteUnpackedRecord(r1);
}
pSorter->aTree[iOut] = iRes;
return SQLITE_OK;
}
/*
** Initialize the temporary index cursor just opened as a sorter cursor.
*/
int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
assert( pCsr->pKeyInfo && pCsr->pBt );
pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter));
return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM);
}
/*
** Free any cursor components allocated by sqlite3VdbeSorterXXX routines.
*/
void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
VdbeSorter *pSorter = pCsr->pSorter;
if( pSorter ){
if( pSorter->aIter ){
int i;
for(i=0; i<pSorter->nTree; i++){
vdbeSorterIterZero(db, &pSorter->aIter[i]);
}
sqlite3DbFree(db, pSorter->aIter);
}
if( pSorter->pTemp1 ){
sqlite3OsCloseFree(pSorter->pTemp1);
}
sqlite3DbFree(db, pSorter);
pCsr->pSorter = 0;
}
}
/*
** Allocate space for a file-handle and open a temporary file. If successful,
** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK.
** Otherwise, set *ppFile to 0 and return an SQLite error code.
*/
static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){
int dummy;
return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile,
SQLITE_OPEN_TEMP_DB |
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy
);
}
/*
** Write the current contents of the b-tree to a PMA. Return SQLITE_OK
** if successful, or an SQLite error code otherwise.
**
** The format of a PMA is:
**
** * A varint. This varint contains the total number of bytes of content
** in the PMA (not including the varint itself).
**
** * One or more records packed end-to-end in order of ascending keys.
** Each record consists of a varint followed by a blob of data (the
** key). The varint is the number of bytes in the blob of data.
*/
static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
int rc = SQLITE_OK; /* Return code */
VdbeSorter *pSorter = pCsr->pSorter;
int res = 0;
rc = sqlite3BtreeFirst(pCsr->pCursor, &res);
if( rc!=SQLITE_OK || res ) return rc;
assert( pSorter->nBtree>0 );
/* If the first temporary PMA file has not been opened, open it now. */
if( pSorter->pTemp1==0 ){
rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1);
assert( rc!=SQLITE_OK || pSorter->pTemp1 );
assert( pSorter->iWriteOff==0 );
assert( pSorter->nPMA==0 );
}
if( rc==SQLITE_OK ){
i64 iWriteOff = pSorter->iWriteOff;
void *aMalloc = 0; /* Array used to hold a single record */
int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */
pSorter->nPMA++;
for(
rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff);
rc==SQLITE_OK && res==0;
rc = sqlite3BtreeNext(pCsr->pCursor, &res)
){
i64 nKey; /* Size of this key in bytes */
/* Write the size of the record in bytes to the output file */
(void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey);
rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff);
/* Make sure the aMalloc[] buffer is large enough for the record */
if( rc==SQLITE_OK && nKey>nMalloc ){
aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey);
if( !aMalloc ){
rc = SQLITE_NOMEM;
}else{
nMalloc = nKey;
}
}
/* Write the record itself to the output file */
if( rc==SQLITE_OK ){
rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
if( rc==SQLITE_OK ){
rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
iWriteOff += nKey;
}
}
if( rc!=SQLITE_OK ) break;
}
/* This assert verifies that unless an error has occurred, the size of
** the PMA on disk is the same as the expected size stored in
** pSorter->nBtree. */
assert( rc!=SQLITE_OK || pSorter->nBtree==(
iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree)
));
pSorter->iWriteOff = iWriteOff;
sqlite3DbFree(db, aMalloc);
}
pSorter->nBtree = 0;
return rc;
}
/*
** This function is called on a sorter cursor by the VDBE before each row
** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in
** bytes, about to be inserted.
**
** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr
** is large enough, its contents are written to a sorted PMA on disk and the
** tree emptied. This prevents the b-tree (which must be small enough to
** fit entirely in the cache in order to support efficient inserts) from
** growing too large.
**
** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK.
*/
int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){
int rc = SQLITE_OK; /* Return code */
VdbeSorter *pSorter = pCsr->pSorter;
if( pSorter ){
Pager *pPager = sqlite3BtreePager(pCsr->pBt);
int nPage; /* Current size of temporary file in pages */
/* Determine how many pages the temporary b-tree has grown to */
sqlite3PagerPagecount(pPager, &nPage);
/* If pSorter->nWorking is still zero, but the temporary file has been
** created in the file-system, then the most recent insert into the
** current b-tree segment probably caused the cache to overflow (it is
** also possible that sqlite3_release_memory() was called). So set the
** size of the working set to a little less than the current size of the
** file in pages. */
if( pSorter->nWorking==0 && sqlite3PagerFile(pPager)->pMethods ){
pSorter->nWorking = nPage-5;
if( pSorter->nWorking<SORTER_MIN_WORKING ){
pSorter->nWorking = SORTER_MIN_WORKING;
}
}
/* If the number of pages used by the current b-tree segment is greater
** than the size of the working set (VdbeSorter.nWorking), start a new
** segment b-tree. */
if( pSorter->nWorking && nPage>=pSorter->nWorking ){
BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */
int iRoot; /* Root page of new tree */
/* Copy the current contents of the b-tree into a PMA in sorted order.
** Close the currently open b-tree cursor. */
rc = vdbeSorterBtreeToPMA(db, pCsr);
sqlite3BtreeCloseCursor(p);
if( rc==SQLITE_OK ){
rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0);
#ifdef SQLITE_DEBUG
sqlite3PagerPagecount(pPager, &nPage);
assert( rc!=SQLITE_OK || nPage==1 );
#endif
}
if( rc==SQLITE_OK ){
rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY);
}
if( rc==SQLITE_OK ){
assert( iRoot==2 );
rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p);
}
}
pSorter->nBtree += sqlite3VarintLen(nKey) + nKey;
}
return rc;
}
/*
** Helper function for sqlite3VdbeSorterRewind().
*/
static int vdbeSorterInitMerge(
sqlite3 *db, /* Database handle */
VdbeCursor *pCsr, /* Cursor handle for this sorter */
i64 *pnByte /* Sum of bytes in all opened PMAs */
){
VdbeSorter *pSorter = pCsr->pSorter;
int rc = SQLITE_OK; /* Return code */
int i; /* Used to iterator through aIter[] */
i64 nByte = 0; /* Total bytes in all opened PMAs */
/* Initialize the iterators. */
for(i=0; rc==SQLITE_OK && i<SORTER_MAX_MERGE_COUNT; i++){
VdbeSorterIter *pIter = &pSorter->aIter[i];
rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte);
pSorter->iReadOff = pIter->iEof;
assert( pSorter->iReadOff<=pSorter->iWriteOff );
if( pSorter->iReadOff>=pSorter->iWriteOff ) break;
}
/* Initialize the aTree[] array. */
for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){
rc = vdbeSorterDoCompare(pCsr, i);
}
*pnByte = nByte;
return rc;
}
/*
** Once the sorter has been populated, this function is called to prepare
** for iterating through its contents in sorted order.
*/
int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
VdbeSorter *pSorter = pCsr->pSorter;
int rc; /* Return code */
sqlite3_file *pTemp2 = 0; /* Second temp file to use */
i64 iWrite2 = 0; /* Write offset for pTemp2 */
int nIter; /* Number of iterators used */
int nByte; /* Bytes of space required for aIter/aTree */
int N = 2; /* Power of 2 >= nIter */
assert( pSorter );
/* Write the current b-tree to a PMA. Close the b-tree cursor. */
rc = vdbeSorterBtreeToPMA(db, pCsr);
sqlite3BtreeCloseCursor(pCsr->pCursor);
if( rc!=SQLITE_OK ) return rc;
if( pSorter->nPMA==0 ){
*pbEof = 1;
return SQLITE_OK;
}
/* Allocate space for aIter[] and aTree[]. */
nIter = pSorter->nPMA;
if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT;
assert( nIter>0 );
while( N<nIter ) N += N;
nByte = N * (sizeof(int) + sizeof(VdbeSorterIter));
pSorter->aIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte);
if( !pSorter->aIter ) return SQLITE_NOMEM;
pSorter->aTree = (int *)&pSorter->aIter[N];
pSorter->nTree = N;
do {
int iNew; /* Index of new, merged, PMA */
for(iNew=0;
rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNT<pSorter->nPMA;
iNew++
){
i64 nWrite; /* Number of bytes in new PMA */
/* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1,
** initialize an iterator for each of them and break out of the loop.
** These iterators will be incrementally merged as the VDBE layer calls
** sqlite3VdbeSorterNext().
**
** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs,
** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs
** are merged into a single PMA that is written to file pTemp2.
*/
rc = vdbeSorterInitMerge(db, pCsr, &nWrite);
assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile );
if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){
break;
}
/* Open the second temp file, if it is not already open. */
if( pTemp2==0 ){
assert( iWrite2==0 );
rc = vdbeSorterOpenTempFile(db, &pTemp2);
}
if( rc==SQLITE_OK ){
rc = vdbeSorterWriteVarint(pTemp2, nWrite, &iWrite2);
}
if( rc==SQLITE_OK ){
int bEof = 0;
while( rc==SQLITE_OK && bEof==0 ){
int nByte;
VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ];
assert( pIter->pFile );
nByte = pIter->nKey + sqlite3VarintLen(pIter->nKey);
rc = sqlite3OsWrite(pTemp2, pIter->aAlloc, nByte, iWrite2);
iWrite2 += nByte;
if( rc==SQLITE_OK ){
rc = sqlite3VdbeSorterNext(db, pCsr, &bEof);
}
}
}
}
if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){
break;
}else{
sqlite3_file *pTmp = pSorter->pTemp1;
pSorter->nPMA = iNew;
pSorter->pTemp1 = pTemp2;
pTemp2 = pTmp;
pSorter->iWriteOff = iWrite2;
pSorter->iReadOff = 0;
iWrite2 = 0;
}
}while( rc==SQLITE_OK );
if( pTemp2 ){
sqlite3OsCloseFree(pTemp2);
}
*pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0);
return rc;
}
/*
** Advance to the next element in the sorter.
*/
int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
VdbeSorter *pSorter = pCsr->pSorter;
int iPrev = pSorter->aTree[1]; /* Index of iterator to advance */
int i; /* Index of aTree[] to recalculate */
int rc; /* Return code */
rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]);
for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){
rc = vdbeSorterDoCompare(pCsr, i);
}
*pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0);
return rc;
}
/*
** Copy the current sorter key into the memory cell pOut.
*/
int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){
VdbeSorter *pSorter = pCsr->pSorter;
VdbeSorterIter *pIter;
pIter = &pSorter->aIter[ pSorter->aTree[1] ];
/* Coverage testing note: As things are currently, this call will always
** succeed. This is because the memory cell passed by the VDBE layer
** happens to be the same one as was used to assemble the keys before they
** were passed to the sorter - meaning it is always large enough for the
** largest key. But this could change very easily, so we leave the call
** to sqlite3VdbeMemGrow() in. */
if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){
return SQLITE_NOMEM;
}
pOut->n = pIter->nKey;
MemSetTypeFlag(pOut, MEM_Blob);
memcpy(pOut->z, pIter->aKey, pIter->nKey);
return SQLITE_OK;
}
#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */
test/index4.test
+112
View File
@@ -0,0 +1,112 @@
# 2011 July 9
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. The
# focus of this file is testing the CREATE INDEX statement.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix index4
do_execsql_test 1.1 {
BEGIN;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES(randomblob(102));
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 2
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 4
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 8
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 16
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 32
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 64
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 128
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 256
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 512
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 1024
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 2048
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 4096
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 8192
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 16384
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 32768
INSERT INTO t1 SELECT randomblob(102) FROM t1; -- 65536
COMMIT;
}
do_execsql_test 1.2 {
CREATE INDEX i1 ON t1(x);
}
do_execsql_test 1.3 {
PRAGMA integrity_check
} {ok}
# The same test again - this time with limited memory.
#
ifcapable memorymanage {
set soft_limit [sqlite3_soft_heap_limit 50000]
db close
sqlite3 db test.db
do_execsql_test 1.4 {
PRAGMA cache_size = 10;
CREATE INDEX i2 ON t1(x);
}
do_execsql_test 1.5 {
PRAGMA integrity_check
} {ok}
sqlite3_soft_heap_limit $soft_limit
}
do_execsql_test 1.6 {
BEGIN;
DROP TABLE t1;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES('a');
INSERT INTO t1 VALUES('b');
INSERT INTO t1 VALUES('c');
INSERT INTO t1 VALUES('d');
INSERT INTO t1 VALUES('e');
INSERT INTO t1 VALUES('f');
INSERT INTO t1 VALUES('g');
INSERT INTO t1 VALUES(NULL);
INSERT INTO t1 SELECT randomblob(1202) FROM t1; -- 16
INSERT INTO t1 SELECT randomblob(2202) FROM t1; -- 32
INSERT INTO t1 SELECT randomblob(3202) FROM t1; -- 64
INSERT INTO t1 SELECT randomblob(4202) FROM t1; -- 128
INSERT INTO t1 SELECT randomblob(5202) FROM t1; -- 256
COMMIT;
CREATE INDEX i1 ON t1(x);
PRAGMA integrity_check
} {ok}
do_execsql_test 1.7 {
BEGIN;
DROP TABLE t1;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES('a');
COMMIT;
CREATE INDEX i1 ON t1(x);
PRAGMA integrity_check
} {ok}
do_execsql_test 1.8 {
BEGIN;
DROP TABLE t1;
CREATE TABLE t1(x);
COMMIT;
CREATE INDEX i1 ON t1(x);
PRAGMA integrity_check
} {ok}
finish_test
test/indexfault.test
+342
View File
@@ -0,0 +1,342 @@
# 2011 August 08
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
ifcapable !mergesort {
finish_test
return
}
set testprefix indexfault
# Set up the custom fault-injector. This is further configured by using
# different values for $::custom_filter and different implementations
# of Tcl proc [xCustom] for each test case.
#
proc install_custom_faultsim {} {
set ::FAULTSIM(custom) [list \
-injectinstall custom_injectinstall \
-injectstart custom_injectstart \
-injectstop custom_injectstop \
-injecterrlist {{1 {disk I/O error}}} \
-injectuninstall custom_injectuninstall \
]
proc custom_injectinstall {} {
testvfs shmfault -default true
shmfault filter $::custom_filter
shmfault script xCustom
}
proc custom_injectuninstall {} {
catch {db close}
catch {db2 close}
shmfault delete
}
set ::custom_ifail -1
set ::custom_nfail -1
proc custom_injectstart {iFail} {
set ::custom_ifail $iFail
set ::custom_nfail 0
}
proc custom_injectstop {} {
set ::custom_ifail -1
return $::custom_nfail
}
}
proc uninstall_custom_faultsim {} {
unset -nocomplain ::FAULTSIM(custom)
}
#-------------------------------------------------------------------------
# These tests - indexfault-1.* - Build an index on a smallish table with
# all different kinds of fault-injection. The CREATE INDEX is run once
# with default options and once with a 50KB soft-heap-limit.
#
do_execsql_test 1.0 {
BEGIN;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES(randomblob(202));
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 2
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 4
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 8
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 16
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 32
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 64
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 128
INSERT INTO t1 SELECT randomblob(202) FROM t1; -- 256
COMMIT;
}
faultsim_save_and_close
do_faultsim_test 1.1 -prep {
faultsim_restore_and_reopen
} -body {
execsql { CREATE INDEX i1 ON t1(x) }
faultsim_test_result {0 {}}
faultsim_integrity_check
}
ifcapable memorymanage {
set soft_limit [sqlite3_soft_heap_limit 50000]
do_faultsim_test 2.1 -prep {
faultsim_restore_and_reopen
} -body {
execsql { CREATE INDEX i1 ON t1(x) }
faultsim_test_result {0 {}}
}
sqlite3_soft_heap_limit $soft_limit
}
#-------------------------------------------------------------------------
# These are similar to the indexfault-1.* tests, except they create an
# index with more than one column.
#
sqlite3 db test.db
do_execsql_test 2.0 {
BEGIN;
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(t,u,v,w,x,y,z);
INSERT INTO t1 VALUES(
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30)
);
INSERT INTO t1 SELECT
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30) FROM t1; -- 2
INSERT INTO t1 SELECT
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30) FROM t1; -- 4
INSERT INTO t1 SELECT
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30) FROM t1; -- 8
INSERT INTO t1 SELECT
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30) FROM t1; -- 16
INSERT INTO t1 SELECT
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30) FROM t1; -- 32
INSERT INTO t1 SELECT
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30) FROM t1; -- 64
INSERT INTO t1 SELECT
randomblob(30), randomblob(30), randomblob(30), randomblob(30),
randomblob(30), randomblob(30), randomblob(30) FROM t1; -- 128
COMMIT;
}
faultsim_save_and_close
do_faultsim_test 2.1 -prep {
faultsim_restore_and_reopen
} -body {
execsql { CREATE INDEX i1 ON t1(t,u,v,w,x,y,z) }
faultsim_test_result {0 {}}
faultsim_integrity_check
}
ifcapable memorymanage {
set soft_limit [sqlite3_soft_heap_limit 50000]
do_faultsim_test 2.2 -prep {
faultsim_restore_and_reopen
} -body {
execsql { CREATE INDEX i1 ON t1(t,u,v,w,x,y,z) }
faultsim_test_result {0 {}}
}
sqlite3_soft_heap_limit $soft_limit
}
#-------------------------------------------------------------------------
# The following tests - indexfault-2.* - all attempt to build a index
# on table t1 in the main database with injected IO errors. Individual
# test cases work as follows:
#
# 3.1: IO errors injected into xOpen() calls.
# 3.2: As 7.1, but with a low (50KB) soft-heap-limit.
#
# 3.3: IO errors injected into the first 200 write() calls made on the
# second temporary file.
# 3.4: As 7.3, but with a low (50KB) soft-heap-limit.
#
# 3.5: After a certain amount of data has been read from the main database
# file (and written into the temporary b-tree), sqlite3_release_memory()
# is called to free as much memory as possible. This causes the temp
# b-tree to be flushed to disk. So that before its contents can be
# transfered to a PMA they must be read back from disk - creating extra
# opportunities for IO errors.
#
install_custom_faultsim
# Set up a table to build indexes on. Save the setup using the
# [faultsim_save_and_close] mechanism.
#
sqlite3 db test.db
do_execsql_test 3.0 {
BEGIN;
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES(randomblob(11000));
INSERT INTO t1 SELECT randomblob(11001) FROM t1; -- 2
INSERT INTO t1 SELECT randomblob(11002) FROM t1; -- 4
INSERT INTO t1 SELECT randomblob(11003) FROM t1; -- 8
INSERT INTO t1 SELECT randomblob(11004) FROM t1; -- 16
INSERT INTO t1 SELECT randomblob(11005) FROM t1; -- 32
INSERT INTO t1 SELECT randomblob(11006) FROM t1; -- 64
INSERT INTO t1 SELECT randomblob(11007) FROM t1; -- 128
INSERT INTO t1 SELECT randomblob(11008) FROM t1; -- 256
INSERT INTO t1 SELECT randomblob(11009) FROM t1; -- 512
COMMIT;
}
faultsim_save_and_close
set ::custom_filter xOpen
proc xCustom {args} {
incr ::custom_ifail -1
if {$::custom_ifail==0} {
incr ::custom_nfail
return "SQLITE_IOERR"
}
return "SQLITE_OK"
}
do_faultsim_test 3.1 -faults custom -prep {
faultsim_restore_and_reopen
} -body {
execsql { CREATE INDEX i1 ON t1(x) }
faultsim_test_result {0 {}}
}
ifcapable memorymanage {
set soft_limit [sqlite3_soft_heap_limit 50000]
do_faultsim_test 3.2 -faults custom -prep {
faultsim_restore_and_reopen
} -body {
execsql { CREATE INDEX i1 ON t1(x) }
faultsim_test_result {0 {}}
}
sqlite3_soft_heap_limit $soft_limit
}
set ::custom_filter {xOpen xWrite}
proc xCustom {method args} {
if {$method == "xOpen"} {
if {[lindex $args 0] == ""} {
incr ::nTmpOpen 1
if {$::nTmpOpen == 3} { return "failme" }
}
return "SQLITE_OK"
}
if {$::custom_ifail<200 && [lindex $args 1] == "failme"} {
incr ::custom_ifail -1
if {$::custom_ifail==0} {
incr ::custom_nfail
return "SQLITE_IOERR"
}
}
return "SQLITE_OK"
}
do_faultsim_test 3.3 -faults custom -prep {
faultsim_restore_and_reopen
set ::nTmpOpen 0
} -body {
execsql { CREATE INDEX i1 ON t1(x) }
faultsim_test_result {0 {}}
}
ifcapable memorymanage {
set soft_limit [sqlite3_soft_heap_limit 50000]
do_faultsim_test 3.4 -faults custom -prep {
faultsim_restore_and_reopen
set ::nTmpOpen 0
} -body {
execsql { CREATE INDEX i1 ON t1(x) }
faultsim_test_result {0 {}}
}
sqlite3_soft_heap_limit $soft_limit
}
uninstall_custom_faultsim
#-------------------------------------------------------------------------
# Test 4: After a certain amount of data has been read from the main database
# file (and written into the temporary b-tree), sqlite3_release_memory() is
# called to free as much memory as possible. This causes the temp b-tree to be
# flushed to disk. So that before its contents can be transfered to a PMA they
# must be read back from disk - creating extra opportunities for IO errors.
#
install_custom_faultsim
catch { db close }
forcedelete test.db
sqlite3 db test.db
do_execsql_test 4.0 {
BEGIN;
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES(randomblob(11000));
INSERT INTO t1 SELECT randomblob(11001) FROM t1; -- 2
INSERT INTO t1 SELECT randomblob(11002) FROM t1; -- 4
INSERT INTO t1 SELECT randomblob(11003) FROM t1; -- 8
INSERT INTO t1 SELECT randomblob(11004) FROM t1; -- 16
INSERT INTO t1 SELECT randomblob(11005) FROM t1; -- 32
INSERT INTO t1 SELECT randomblob(11005) FROM t1; -- 64
COMMIT;
}
faultsim_save_and_close
testvfs tvfs
tvfs script xRead
tvfs filter xRead
set ::nRead 0
proc xRead {method file args} {
if {[file tail $file] == "test.db"} { incr ::nRead }
}
do_test 4.1 {
sqlite3 db test.db -vfs tvfs
execsql { CREATE INDEX i1 ON t1(x) }
} {}
db close
tvfs delete
set ::custom_filter xRead
proc xCustom {method file args} {
incr ::nReadCall
if {$::nReadCall >= ($::nRead/5)} {
if {$::nReadCall == ($::nRead/5)} {
set nByte [sqlite3_release_memory [expr 64*1024*1024]]
sqlite3_soft_heap_limit 20000
}
if {$file == ""} {
incr ::custom_ifail -1
if {$::custom_ifail==0} {
incr ::custom_nfail
return "SQLITE_IOERR"
}
}
}
return "SQLITE_OK"
}
do_faultsim_test 4.2 -faults custom -prep {
faultsim_restore_and_reopen
set ::nReadCall 0
sqlite3_soft_heap_limit 0
} -body {
execsql { CREATE INDEX i1 ON t1(x) }
faultsim_test_result {0 {}}
}
uninstall_custom_faultsim
finish_test
test/permutations.test
+1 -1
View File
@@ -111,7 +111,7 @@ set allquicktests [test_set $alltests -exclude {
thread003.test thread004.test thread005.test trans2.test vacuum3.test
incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
vtab_err.test walslow.test walcrash.test
walthread.test rtree3.test
walthread.test rtree3.test indexfault.test
}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
test/wal2.test
+1
View File
@@ -606,6 +606,7 @@ do_test wal2-6.4.1 {
testvfs tvfs
tvfs script tvfs_cb
sqlite3 db test.db -vfs tvfs
set {} {}
} {}
set RECOVERY {
tool/mksqlite3c.tcl
+1
View File
@@ -259,6 +259,7 @@ foreach file {
vdbetrace.c
vdbe.c
vdbeblob.c
vdbesort.c
journal.c
memjournal.c