Try to use mmap() to speed access to the database file on windows, linux,

and mac.

FossilOrigin-Name: fff2be60779571c8fb89158db976ec3755e9a223
This commit is contained in:
drh
2013-04-04 00:40:17 +00:00
40 changed files with 1693 additions and 171 deletions
+50
View File
@@ -0,0 +1,50 @@
# 2013 April 02
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains fault injection tests designed to test the btree.c
# module.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
set testprefix btreefault
do_test 1-pre1 {
execsql {
PRAGMA auto_vacuum = incremental;
PRAGMA journal_mode = DELETE;
CREATE TABLE t1(a PRIMARY KEY, b);
INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
DELETE FROM t1 WHERE rowid%2;
}
faultsim_save_and_close
} {}
do_faultsim_test 1 -prep {
faultsim_restore_and_reopen
set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY]
sqlite3_step $::STMT
sqlite3_step $::STMT
} -body {
execsql { PRAGMA incremental_vacuum = 10 }
} -test {
sqlite3_finalize $::STMT
faultsim_test_result {0 {}}
faultsim_integrity_check
}
finish_test
+1
View File
@@ -40,6 +40,7 @@ proc db_write {db {reset 0}} {
do_test 1.1 {
db close
sqlite3 db test.db
execsql { PRAGMA mmap_limit = 0 }
expr {[file size test.db] / 1024}
} 6
+8
View File
@@ -25,6 +25,14 @@ ifcapable {!pager_pragmas} {
return
}
# Tests in this file verify that locking_mode=exclusive causes SQLite to
# use cached pages even if the database is changed on disk. This doesn't
# work with mmap.
if {[permutation]!="nommap"} {
finish_test
return
}
# This module does not work right if the cache spills at unexpected
# moments. So disable the soft-heap-limit.
#
+7 -5
View File
@@ -1273,11 +1273,13 @@ do_test func-29.3 {
sqlite3_db_status db CACHE_MISS 1
db eval {SELECT typeof(+x) FROM t29 ORDER BY id}
} {integer null real blob text}
do_test func-29.4 {
set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1]
if {$x>100} {set x many}
set x
} {many}
if {[permutation] == "nommap"} {
do_test func-29.4 {
set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1]
if {$x>100} {set x many}
set x
} {many}
}
do_test func-29.5 {
db close
sqlite3 db test.db
+4
View File
@@ -123,6 +123,7 @@ foreach AutoVacuumMode [list 0 1] {
forcedelete test.db test.db-journal
sqlite3 db test.db
execsql "PRAGMA mmap_limit = 0"
execsql "PRAGMA auto_vacuum = $AutoVacuumMode"
do_test incrblob-2.$AutoVacuumMode.1 {
@@ -149,6 +150,7 @@ foreach AutoVacuumMode [list 0 1] {
# Open and close the db to make sure the page cache is empty.
db close
sqlite3 db test.db
execsql "PRAGMA mmap_limit = 0"
# Read the last 20 bytes of the blob via a blob handle.
set ::blob [db incrblob blobs v 1]
@@ -171,6 +173,7 @@ foreach AutoVacuumMode [list 0 1] {
# Open and close the db to make sure the page cache is empty.
db close
sqlite3 db test.db
execsql "PRAGMA mmap_limit = 0"
# Write the second-to-last 20 bytes of the blob via a blob handle.
#
@@ -200,6 +203,7 @@ foreach AutoVacuumMode [list 0 1] {
# Open and close the db to make sure the page cache is empty.
db close
sqlite3 db test.db
execsql { PRAGMA mmap_limit = 0 }
execsql { SELECT i FROM blobs }
} {45}
+1 -1
View File
@@ -264,7 +264,7 @@ proc faultsim_test_result_int {args} {
set t [list $testrc $testresult]
set r $args
if { ($testnfail==0 && $t != [lindex $r 0]) || [lsearch $r $t]<0 } {
error "nfail=$testnfail rc=$testrc result=$testresult"
error "nfail=$testnfail rc=$testrc result=$testresult list=$r"
}
}
+250
View File
@@ -0,0 +1,250 @@
# 2013 March 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
set testprefix mmap1
proc nRead {db} {
set bt [btree_from_db $db]
db_enter $db
array set stats [btree_pager_stats $bt]
db_leave $db
# puts [array get stats]
return $stats(read)
}
foreach {t mmap_limit nRead c2init} {
1.1 { PRAGMA mmap_limit = 67108864 } 4 {}
1.2 { PRAGMA mmap_limit = 53248 } 150 {}
1.3 { PRAGMA mmap_limit = 0 } 344 {}
1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 }
1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 }
1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 }
} {
do_multiclient_test tn {
sql1 {PRAGMA page_size=1024}
sql1 $mmap_limit
sql2 $c2init
code2 {
set ::rcnt 0
proc rblob {n} {
set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF]
set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]]
string range [string repeat $str [expr $n/4]] 1 $n
}
db2 func rblob rblob
}
sql2 {
PRAGMA page_size=1024;
PRAGMA auto_vacuum = 1;
CREATE TABLE t1(a, b, UNIQUE(a, b));
INSERT INTO t1 VALUES(rblob(500), rblob(500));
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32
}
do_test $t.$tn.1 {
sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
} {32 ok 77}
# Have connection 2 shrink the file. Check connection 1 can still read it.
sql2 { DELETE FROM t1 WHERE rowid%2; }
do_test $t.$tn.2 {
sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
} {16 ok 42}
# Have connection 2 grow the file. Check connection 1 can still read it.
sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 }
do_test $t.$tn.3 {
sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
} {32 ok 79}
# Have connection 2 grow the file again. Check connection 1 is still ok.
sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 }
do_test $t.$tn.4 {
sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
} {64 ok 149}
# Check that the number of pages read by connection 1 indicates that the
# "PRAGMA mmap_limit" command worked.
do_test $t.$tn.5 { nRead db } $nRead
}
}
set ::rcnt 0
proc rblob {n} {
set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF]
set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]]
string range [string repeat $str [expr $n/4]] 1 $n
}
reset_db
db func rblob rblob
do_execsql_test 2.1 {
PRAGMA auto_vacuum = 1;
PRAGMA mmap_limit = 67108864;
PRAGMA journal_mode = wal;
CREATE TABLE t1(a, b, UNIQUE(a, b));
INSERT INTO t1 VALUES(rblob(500), rblob(500));
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32
PRAGMA wal_checkpoint;
} {67108864 wal 0 103 103}
do_execsql_test 2.2 {
PRAGMA auto_vacuum;
SELECT count(*) FROM t1;
} {1 32}
do_test 2.3 {
sqlite3 db2 test.db
db2 func rblob rblob
db2 eval {
DELETE FROM t1 WHERE (rowid%4);
PRAGMA wal_checkpoint;
}
db2 eval {
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16
SELECT count(*) FROM t1;
}
} {16}
do_execsql_test 2.4 {
PRAGMA wal_checkpoint;
} {0 24 24}
db2 close
reset_db
db func rblob rblob
do_execsql_test 3.1 {
PRAGMA auto_vacuum = 1;
CREATE TABLE t1(a, b, UNIQUE(a, b));
INSERT INTO t1 VALUES(rblob(500), rblob(500));
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4
INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8
CREATE TABLE t2(a, b, UNIQUE(a, b));
INSERT INTO t2 SELECT * FROM t1;
} {}
do_test 3.2 {
set nRow 0
db eval {SELECT * FROM t2 ORDER BY a, b} {
if {$nRow==4} { db eval { DELETE FROM t1 } }
incr nRow
}
set nRow
} {8}
#-------------------------------------------------------------------------
# Ensure that existing cursors using xFetch() pages see changes made
# to rows using the incrblob API.
#
reset_db
set aaa [string repeat a 400]
set bbb [string repeat b 400]
set ccc [string repeat c 400]
set ddd [string repeat d 400]
set eee [string repeat e 400]
do_execsql_test 4.1 {
PRAGMA page_size = 1024;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES($aaa);
INSERT INTO t1 VALUES($bbb);
INSERT INTO t1 VALUES($ccc);
INSERT INTO t1 VALUES($ddd);
SELECT * FROM t1;
BEGIN;
} [list $aaa $bbb $ccc $ddd]
do_test 4.2 {
set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy]
sqlite3_step $::STMT
sqlite3_column_text $::STMT 0
} $aaa
do_test 4.3 {
foreach r {2 3 4} {
set fd [db incrblob t1 x $r]
puts -nonewline $fd $eee
close $fd
}
set res [list]
while {"SQLITE_ROW" == [sqlite3_step $::STMT]} {
lappend res [sqlite3_column_text $::STMT 0]
}
set res
} [list $eee $eee $eee]
do_test 4.4 {
sqlite3_finalize $::STMT
} SQLITE_OK
do_execsql_test 4.5 { COMMIT }
#-------------------------------------------------------------------------
# Ensure that existing cursors holding xFetch() references are not
# confused if those pages are moved to make way for the root page of a
# new table or index.
#
reset_db
do_execsql_test 5.1 {
PRAGMA auto_vacuum = 2;
PRAGMA page_size = 1024;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES($aaa);
INSERT INTO t1 VALUES($bbb);
INSERT INTO t1 VALUES($ccc);
INSERT INTO t1 VALUES($ddd);
PRAGMA auto_vacuum;
SELECT * FROM t1;
} [list 2 $aaa $bbb $ccc $ddd]
do_test 5.2 {
set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy]
sqlite3_step $::STMT
sqlite3_column_text $::STMT 0
} $aaa
do_execsql_test 5.3 {
CREATE TABLE t2(x);
INSERT INTO t2 VALUES('tricked you!');
INSERT INTO t2 VALUES('tricked you!');
}
do_test 5.4 {
sqlite3_step $::STMT
sqlite3_column_text $::STMT 0
} $bbb
do_test 5.5 {
sqlite3_finalize $::STMT
} SQLITE_OK
finish_test
+82
View File
@@ -0,0 +1,82 @@
# 2013 March 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests the effect of the mmap() or mremap() system calls
# returning an error on the library.
#
# If either mmap() or mremap() fails, SQLite should log an error
# message, then continue accessing the database using read() and
# write() exclusively.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix mmap2
if {$::tcl_platform(platform)!="unix" || [test_syscall defaultvfs] != "unix"} {
finish_test
return
}
db close
sqlite3_shutdown
test_sqlite3_log xLog
proc xLog {error_code msg} {
if {[string match os_unix.c* $msg]} {
lappend ::log $msg
}
}
foreach syscall {mmap mremap} {
test_syscall uninstall
if {[catch {test_syscall install $syscall}]} continue
for {set i 1} {$i < 20} {incr i} {
reset_db
test_syscall fault $i 1
test_syscall errno $syscall ENOMEM
set ::log ""
do_execsql_test 1.$syscall.$i.1 {
CREATE TABLE t1(a, b, UNIQUE(a, b));
INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000));
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
}
set nFail [test_syscall fault 0 0]
do_execsql_test 1.$syscall.$i.2 {
SELECT count(*) FROM t1;
PRAGMA integrity_check;
} {64 ok}
do_test 1.$syscall.$i.3 {
expr {$nFail==0 || $nFail==1}
} {1}
do_test 1.$syscall.$i.4.nFail=$nFail {
regexp ".*${syscall}.*" $::log
} [expr $nFail>0]
}
}
db close
test_syscall uninstall
sqlite3_shutdown
test_sqlite3_log
sqlite3_initialize
finish_test
+57
View File
@@ -1246,5 +1246,62 @@ do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep {
faultsim_integrity_check
}
#-------------------------------------------------------------------------
#
do_test pagerfault-28-pre {
faultsim_delete_and_reopen
db func a_string a_string
execsql {
PRAGMA page_size = 512;
PRAGMA journal_mode = wal;
PRAGMA wal_autocheckpoint = 0;
PRAGMA cache_size = 100000;
BEGIN;
CREATE TABLE t2(a UNIQUE, b UNIQUE);
INSERT INTO t2 VALUES( a_string(800), a_string(800) );
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2;
COMMIT;
CREATE TABLE t1(a PRIMARY KEY, b);
}
expr {[file size test.db-shm] >= 96*1024}
} {1}
faultsim_save_and_close
do_faultsim_test pagerfault-28 -faults oom* -prep {
faultsim_restore_and_reopen
execsql { PRAGMA mmap_limit=0 }
sqlite3 db2 test.db
db2 eval { SELECT count(*) FROM t2 }
db func a_string a_string
execsql {
BEGIN;
INSERT INTO t1 VALUES(a_string(2000), a_string(2000));
INSERT INTO t1 VALUES(a_string(2000), a_string(2000));
}
set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY]
sqlite3_step $::STMT
} -body {
execsql { ROLLBACK }
} -test {
db2 close
sqlite3_finalize $::STMT
catchsql { ROLLBACK }
faultsim_integrity_check
}
finish_test
+1 -1
View File
@@ -92,7 +92,7 @@ do_test pageropt-1.5 {
pagercount_sql {
SELECT hex(x) FROM t1
}
} [list 6 0 0 $blobcontent]
} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent]
do_test pageropt-1.6 {
pagercount_sql {
SELECT hex(x) FROM t1
+8
View File
@@ -138,6 +138,14 @@ test_suite "veryquick" -prefix "" -description {
test_set $allquicktests -exclude *malloc* *ioerr* *fault*
]
test_suite "nommap" -prefix "nomm-" -description {
Similar to veryquick. Except with memory mapping disabled.
} -presql {
pragma mmap_limit = 0;
} -files [
test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test
]
test_suite "valgrind" -prefix "" -description {
Run the "veryquick" test suite with a couple of multi-process tests (that
fail under valgrind) omitted.
+2 -1
View File
@@ -24,6 +24,8 @@ set testdir [file dirname $argv0]
source $testdir/tester.tcl
speed_trial_init speed1
sqlite3_memdebug_vfs_oom_test 0
# Set a uniform random seed
expr srand(0)
@@ -78,7 +80,6 @@ do_test speed1p-1.0 {
}
} {i2a i2b t1 t2}
# 50000 INSERTs on an unindexed table
#
set list {}
+1 -1
View File
@@ -60,7 +60,7 @@ foreach s {
open close access getcwd stat fstat ftruncate
fcntl read pread write pwrite fchmod fallocate
pread64 pwrite64 unlink openDirectory mkdir rmdir
statvfs fchown umask
statvfs fchown umask mmap munmap mremap
} {
if {[test_syscall exists $s]} {lappend syscall_list $s}
}
+31 -1
View File
@@ -243,5 +243,35 @@ do_faultsim_test 3 -faults vfsfault-* -prep {
faultsim_test_result {0 20000}
}
finish_test
#-------------------------------------------------------------------------
# Test errors in mmap().
#
proc vfsfault_install {} {
test_syscall reset
test_syscall install {mmap}
}
faultsim_delete_and_reopen
execsql {
CREATE TABLE t1(a, b);
INSERT INTO t1 VALUES(1, 2);
}
faultsim_save_and_close
do_faultsim_test 4 -faults vfsfault-* -prep {
faultsim_restore_and_reopen
file_control_chunksize_test db main 8192
execsql {
PRAGMA mmap_limit = 1000000;
}
} -body {
test_syscall errno mmap EACCES
execsql {
SELECT * FROM t1;
}
} -test {
faultsim_test_result {0 {1 2}} {1 {disk I/O error}}
}
finish_test
+25 -3
View File
@@ -727,6 +727,8 @@ do_test wal-11.9 {
list [expr [file size test.db]/1024] [log_deleted test.db-wal]
} {37 1}
sqlite3_wal db test.db
set nWal 39
if {[permutation]=="nommap"} {set nWal 37}
do_test wal-11.10 {
execsql {
PRAGMA cache_size = 10;
@@ -735,7 +737,7 @@ do_test wal-11.10 {
SELECT count(*) FROM t1;
}
list [expr [file size test.db]/1024] [file size test.db-wal]
} [list 37 [wal_file_size 37 1024]]
} [list 37 [wal_file_size $nWal 1024]]
do_test wal-11.11 {
execsql {
SELECT count(*) FROM t1;
@@ -745,7 +747,7 @@ do_test wal-11.11 {
} {32 16}
do_test wal-11.12 {
list [expr [file size test.db]/1024] [file size test.db-wal]
} [list 37 [wal_file_size 37 1024]]
} [list 37 [wal_file_size $nWal 1024]]
do_test wal-11.13 {
execsql {
INSERT INTO t1 VALUES( blob(900) );
@@ -755,7 +757,7 @@ do_test wal-11.13 {
} {17 ok}
do_test wal-11.14 {
list [expr [file size test.db]/1024] [file size test.db-wal]
} [list 37 [wal_file_size 37 1024]]
} [list 37 [wal_file_size $nWal 1024]]
#-------------------------------------------------------------------------
@@ -1567,6 +1569,26 @@ ifcapable autovacuum {
} [wal_file_size 1 1024]
}
reset_db
do_test 25 {
sqlite3 db test.db
execsql {
CREATE TABLE t1(x, y);
PRAGMA journal_mode = WAL;
INSERT INTO t1 VALUES(1, 2);
}
execsql {
BEGIN;
CREATE TABLE t2(a, b);
}
hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192]
catchsql ROLLBACK
} {0 {}}
db close
sqlite3_shutdown
test_sqlite3_log
+9 -2
View File
@@ -235,7 +235,15 @@ foreach {testprefix do_wal_checkpoint} {
do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {}
do_test 2.3.$tn.6 { file_page_counts } {1 4 1 4}
do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 4 3}
do_test 2.3.$tn.8 { file_page_counts } {1 4 2 4}
# The checkpoint above only writes page 1 of the db file. The other
# page (page 2) is locked by the read-transaction opened by the
# [sql2] commmand above. So normally, the db is 1 page in size here.
# However, in mmap() mode, the db is pre-allocated to 2 pages at the
# start of the checkpoint, even though page 2 cannot be written.
set nDb 2
if {[permutation]=="no-mmap"} {set nDb 1}
do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4]
}
# Check that checkpoints block on the correct locks. And respond correctly
@@ -343,4 +351,3 @@ foreach {testprefix do_wal_checkpoint} {
finish_test