paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add the optional (and experimental) mmap() memory allocator in the

Browse Source 
mem4.c module. (CVS 4581)

FossilOrigin-Name: cfd683ac80fd043343e0f0af90805058daa3818d
This commit is contained in:
drh
2007-11-29 18:36:49 +00:00
parent 21de2e755c
commit ace03d1b3a
8 changed files with 428 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.in
+5 -1
View File
@@ -123,7 +123,7 @@ TCC += -DSQLITE_OMIT_LOAD_EXTENSION=1
LIBOBJ = alter.lo analyze.lo attach.lo auth.lo btmutex.lo btree.lo build.lo \
callback.lo complete.lo date.lo \
delete.lo expr.lo func.lo hash.lo journal.lo insert.lo loadext.lo \
main.lo malloc.lo mem1.lo mem2.lo mem3.lo mutex.lo \
main.lo malloc.lo mem1.lo mem2.lo mem3.lo mem4.lo mutex.lo \
mutex_os2.lo mutex_unix.lo mutex_w32.lo \
opcodes.lo os.lo os_unix.lo os_win.lo os_os2.lo \
pager.lo parse.lo pragma.lo prepare.lo printf.lo random.lo \
@@ -160,6 +160,7 @@ SRC = \
$(TOP)/src/mem1.c \
$(TOP)/src/mem2.c \
$(TOP)/src/mem3.c \
$(TOP)/src/mem4.c \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex_os2.c \
$(TOP)/src/mutex_unix.c \
@@ -415,6 +416,9 @@ mem2.lo: $(TOP)/src/mem2.c $(HDR)
mem3.lo: $(TOP)/src/mem3.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem3.c
mem4.lo: $(TOP)/src/mem4.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem4.c
mutex.lo: $(TOP)/src/mutex.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex.c
main.mk
+2 -1
View File
@@ -51,7 +51,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src
LIBOBJ+= alter.o analyze.o attach.o auth.o btmutex.o btree.o build.o \
callback.o complete.o date.o delete.o \
expr.o func.o hash.o insert.o journal.o loadext.o \
main.o malloc.o mem1.o mem2.o mem3.o mutex.o mutex_os2.o \
main.o malloc.o mem1.o mem2.o mem3.o mem4.o mutex.o mutex_os2.o \
mutex_unix.o mutex_w32.o \
opcodes.o os.o os_os2.o os_unix.o os_win.o \
pager.o parse.o pragma.o prepare.o printf.o random.o \
@@ -107,6 +107,7 @@ SRC = \
$(TOP)/src/mem1.c \
$(TOP)/src/mem2.c \
$(TOP)/src/mem3.c \
$(TOP)/src/mem4.c \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex.h \
$(TOP)/src/mutex_os2.c \
manifest
+12 -11
View File
@@ -1,7 +1,7 @@
C When\susing\san\sindex\sto\sscan\sa\sdatabase\stable,\sread\scolumn\sdata\sfrom\sthe\sindex\sin\spreference\sto\sthe\stable.\sThis\sincreases\sthe\slikelihood\sthat\sthe\stable\swill\snot\sbe\srequired\sat\sall.\s(CVS\s4580)
D 2007-11-29T17:43:28
C Add\sthe\soptional\s(and\sexperimental)\smmap()\smemory\sallocator\sin\sthe\nmem4.c\smodule.\s(CVS\s4581)
D 2007-11-29T18:36:49
F Makefile.arm-wince-mingw32ce-gcc ac5f7b2cef0cd850d6f755ba6ee4ab961b1fadf7
F Makefile.in 35396fd58890420b29edcf27b6c0e2d054862a6b
F Makefile.in d9419c71360931f0711d572e06f726f29656bfc3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
F VERSION 37c652fc2532226ab2fef7184de9500dafa52d7d
@@ -64,7 +64,7 @@ F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
F ext/icu/icu.c 61a345d8126686aa3487aa8d2d0f68abd655f7a4
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
F ltmain.sh 56abb507100ed2d4261f6dd1653dec3cf4066387
F main.mk 297e32b77abde73e5638e048d474a86617bc40af
F main.mk ba3fca8903c6f3ca26cf4fa7a123c0d3eb2fe151
F mkdll.sh 5f8438dcac98e795d7df6529159a1ec566de0183
F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
F mkextw.sh 1a866b53637dab137191341cc875575a5ca110fb
@@ -104,9 +104,10 @@ F src/loadext.c 124e566563d1c03e68e1396cb44df9870612c6e9
F src/main.c 994a6b6914d91dc6dea5012667ec0a52e74d3bca
F src/malloc.c 60e392a4c12c839517f9b0db7b995f825444fb35
F src/md5.c c5fdfa5c2593eaee2e32a5ce6c6927c986eaf217
F src/mem1.c ad348eedd829528e66f4a5aead464d88e6b08d69
F src/mem2.c 2a1da2e8debcfd0097188470f04573107a018116
F src/mem3.c a9857cf92c9e4c889184b2cf1ca1839c801fc942
F src/mem1.c 6d1a11864963d249c67e72ad5f6533b040333880
F src/mem2.c a9400e06b41ad5b5189e8416239833212de2c8c7
F src/mem3.c 34ffb9f7dd7645bece4b022c5f48766550b68886
F src/mem4.c 36ecd536a8b7acfe4cbf011353dae6ea68121e40
F src/mutex.c 3259f62c2429967aee6dc112117a6d2f499ef061
F src/mutex.h 079fa6fe9da18ceb89e79012c010594c6672addb
F src/mutex_os2.c 7fe4773e98ed74a63b2e54fc557929eb155f6269
@@ -594,7 +595,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
P 56d0e32677744df8570b519fae1c04da4ea4984d
R 36e691e0ba7fd3d68c366765c689ffb2
U danielk1977
Z 64a9f02b9181db1b1e3b35c33464441d
P 061608c72ac0a96eacf3b64d638235e4739f96ba
R 1c1c1bbe1258e7de094daae06d9ba74e
U drh
Z 1c04b0f4f3c446511a85e32381699825
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
061608c72ac0a96eacf3b64d638235e4739f96ba
cfd683ac80fd043343e0f0af90805058daa3818d
src/mem1.c
+3 -2
View File
@@ -12,7 +12,7 @@
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.
**
** $Id: mem1.c,v 1.13 2007/11/05 17:54:17 drh Exp $
** $Id: mem1.c,v 1.14 2007/11/29 18:36:49 drh Exp $
*/
/*
@@ -20,7 +20,8 @@
** used when no other memory allocator is specified using compile-time
** macros.
*/
#if !defined(SQLITE_MEMDEBUG) && !defined(SQLITE_MEMORY_SIZE)
#if !defined(SQLITE_MEMDEBUG) && !defined(SQLITE_MEMORY_SIZE) \
&& !defined(SQLITE_MMAP_HEAP_SIZE)
/*
** We will eventually construct multiple memory allocation subsystems
src/mem2.c
+2 -2
View File
@@ -12,7 +12,7 @@
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.
**
** $Id: mem2.c,v 1.17 2007/11/05 17:54:17 drh Exp $
** $Id: mem2.c,v 1.18 2007/11/29 18:36:49 drh Exp $
*/
/*
@@ -20,7 +20,7 @@
** SQLITE_MEMDEBUG macro is defined and SQLITE_OMIT_MEMORY_ALLOCATION
** is not defined.
*/
#if defined(SQLITE_MEMDEBUG) && !defined(SQLITE_MEMORY_SIZE)
#if defined(SQLITE_MEMDEBUG)
/*
** We will eventually construct multiple memory allocation subsystems
src/mem3.c
+5 -1
View File
@@ -20,7 +20,7 @@
** This version of the memory allocation subsystem is used if
** and only if SQLITE_MEMORY_SIZE is defined.
**
** $Id: mem3.c,v 1.6 2007/11/07 15:13:25 drh Exp $
** $Id: mem3.c,v 1.7 2007/11/29 18:36:49 drh Exp $
*/
/*
@@ -30,6 +30,10 @@
#if defined(SQLITE_MEMORY_SIZE)
#include "sqliteInt.h"
#ifdef SQLITE_MEMDEBUG
# error cannot define both SQLITE_MEMDEBUG and SQLITE_MEMORY_SIZE
#endif
/*
** Maximum size (in Mem3Blocks) of a "small" chunk.
*/
src/mem4.c
+398
View File
@@ -0,0 +1,398 @@
/*
** 2007 August 14
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.
**
** $Id: mem4.c,v 1.1 2007/11/29 18:36:49 drh Exp $
*/
/*
** This version of the memory allocator attempts to obtain memory
** from mmap() if the size of the allocation is close to the size
** of a virtual memory page. If the size of the allocation is different
** from the virtual memory page size, then ordinary malloc() is used.
** Ordinary malloc is also used if space allocated to mmap() is
** exhausted.
**
** Enable this memory allocation by compiling with -DSQLITE_MMAP_HEAP_SIZE=nnn
** where nnn is the maximum number of bytes of mmap-ed memory you want
** to support. This module may choose to use less memory than requested.
**
*/
#if defined(SQLITE_MMAP_HEAP_SIZE)
#if defined(SQLITE_MEMDEBUG) || defined(SQLITE_MEMORY_SIZE)
# error cannot use SQLITE_MMAP_HEAP_SIZE with either SQLITE_MEMDEBUG \
or SQLITE_MEMORY_SIZE
#endif
/*
** This is a test version of the memory allocator that attempts to
** use mmap() and madvise() for allocations and frees of approximately
** the virtual memory page size.
*/
#include <sys/types.h>
#include <sys/mman.h>
#include <errno.h>
#include "sqliteInt.h"
#include <unistd.h>
/*
** All of the static variables used by this module are collected
** into a single structure named "mem". This is to keep the
** static variables organized and to reduce namespace pollution
** when this module is combined with other in the amalgamation.
*/
static struct {
/*
** The alarm callback and its arguments. The mem.mutex lock will
** be held while the callback is running. Recursive calls into
** the memory subsystem are allowed, but no new callbacks will be
** issued. The alarmBusy variable is set to prevent recursive
** callbacks.
*/
sqlite3_int64 alarmThreshold;
void (*alarmCallback)(void*, sqlite3_int64,int);
void *alarmArg;
int alarmBusy;
/*
** Mutex to control access to the memory allocation subsystem.
*/
sqlite3_mutex *mutex;
/*
** Current allocation and high-water mark.
*/
sqlite3_int64 nowUsed;
sqlite3_int64 mxUsed;
/*
** Current allocation and high-water marks for mmap allocated memory.
*/
sqlite3_int64 nowUsedMMap;
sqlite3_int64 mxUsedMMap;
/*
** Size of a single mmap page. Obtained from sysconf().
*/
int szPage;
int mnPage;
/*
** The number of available mmap pages.
*/
int nPage;
/*
** Index of the first free page. 0 means no pages have been freed.
*/
int firstFree;
/* First unused page on the top of the heap.
*/
int firstUnused;
/*
** Bulk memory obtained from from mmap().
*/
char *mmapHeap; /* first byte of the heap */
} mem;
/*
** Enter the mutex mem.mutex. Allocate it if it is not already allocated.
** The mmap() region is initialized the first time this routine is called.
*/
static void memsys4Enter(void){
if( mem.mutex==0 ){
mem.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM);
}
sqlite3_mutex_enter(mem.mutex);
}
/*
** Attempt to free memory to the mmap heap. This only works if
** the pointer p is within the range of memory addresses that
** comprise the mmap heap. Return 1 if the memory was freed
** successfully. Return 0 if the pointer is out of range.
*/
static int mmapFree(void *p){
char *z;
int idx, *a;
if( mem.mmapHeap==MAP_FAILED || mem.nPage==0 ){
return 0;
}
z = (char*)p;
idx = (z - mem.mmapHeap)/mem.szPage;
if( idx<1 || idx>=mem.nPage ){
return 0;
}
a = (int*)mem.mmapHeap;
a[idx] = a[mem.firstFree];
mem.firstFree = idx;
mem.nowUsedMMap -= mem.szPage;
madvise(p, mem.szPage, MADV_DONTNEED);
return 1;
}
/*
** Attempt to allocate nBytes from the mmap heap. Return a pointer
** to the allocated page. Or, return NULL if the allocation fails.
**
** The allocation will fail if nBytes is not the right size.
** Or, the allocation will fail if the mmap heap has been exhausted.
*/
static void *mmapAlloc(int nBytes){
int idx = 0;
if( nBytes>mem.szPage || nBytes<mem.mnPage ){
return 0;
}
if( mem.nPage==0 ){
mem.szPage = sysconf(_SC_PAGE_SIZE);
mem.mnPage = mem.szPage - mem.szPage/10;
mem.nPage = SQLITE_MMAP_HEAP_SIZE/mem.szPage;
if( mem.nPage * sizeof(int) > mem.szPage ){
mem.nPage = mem.szPage/sizeof(int);
}
mem.mmapHeap = mmap(0, mem.szPage*mem.nPage, PROT_WRITE|PROT_READ,
MAP_ANONYMOUS|MAP_SHARED, -1, 0);
if( mem.mmapHeap==MAP_FAILED ){
mem.firstUnused = errno;
}else{
mem.firstUnused = 1;
mem.nowUsedMMap = mem.szPage;
}
}
if( mem.mmapHeap==MAP_FAILED ){
return 0;
}
if( mem.firstFree ){
int idx = mem.firstFree;
int *a = (int*)mem.mmapHeap;
mem.firstFree = a[idx];
}else if( mem.firstUnused<mem.nPage ){
idx = mem.firstUnused++;
}
if( idx ){
mem.nowUsedMMap += mem.szPage;
if( mem.nowUsedMMap>mem.mxUsedMMap ){
mem.mxUsedMMap = mem.nowUsedMMap;
}
return (void*)&mem.mmapHeap[idx*mem.szPage];
}else{
return 0;
}
}
/*
** Release the mmap-ed memory region if it is currently allocated and
** is not in use.
*/
static void mmapUnmap(void){
if( mem.mmapHeap==MAP_FAILED ) return;
if( mem.nPage==0 ) return;
if( mem.nowUsedMMap>mem.szPage ) return;
munmap(mem.mmapHeap, mem.nPage*mem.szPage);
mem.nowUsedMMap = 0;
mem.nPage = 0;
}
/*
** Return the amount of memory currently checked out.
*/
sqlite3_int64 sqlite3_memory_used(void){
sqlite3_int64 n;
memsys4Enter();
n = mem.nowUsed + mem.nowUsedMMap;
sqlite3_mutex_leave(mem.mutex);
return n;
}
/*
** Return the maximum amount of memory that has ever been
** checked out since either the beginning of this process
** or since the most recent reset.
*/
sqlite3_int64 sqlite3_memory_highwater(int resetFlag){
sqlite3_int64 n;
memsys4Enter();
n = mem.mxUsed + mem.mxUsedMMap;
if( resetFlag ){
mem.mxUsed = mem.nowUsed;
mem.mxUsedMMap = mem.nowUsedMMap;
}
sqlite3_mutex_leave(mem.mutex);
return n;
}
/*
** Change the alarm callback
*/
int sqlite3_memory_alarm(
void(*xCallback)(void *pArg, sqlite3_int64 used,int N),
void *pArg,
sqlite3_int64 iThreshold
){
memsys4Enter();
mem.alarmCallback = xCallback;
mem.alarmArg = pArg;
mem.alarmThreshold = iThreshold;
sqlite3_mutex_leave(mem.mutex);
return SQLITE_OK;
}
/*
** Trigger the alarm
*/
static void sqlite3MemsysAlarm(int nByte){
void (*xCallback)(void*,sqlite3_int64,int);
sqlite3_int64 nowUsed;
void *pArg;
if( mem.alarmCallback==0 || mem.alarmBusy ) return;
mem.alarmBusy = 1;
xCallback = mem.alarmCallback;
nowUsed = mem.nowUsed;
pArg = mem.alarmArg;
sqlite3_mutex_leave(mem.mutex);
xCallback(pArg, nowUsed, nByte);
sqlite3_mutex_enter(mem.mutex);
mem.alarmBusy = 0;
}
/*
** Allocate nBytes of memory
*/
static void *memsys4Malloc(int nBytes){
sqlite3_int64 *p = 0;
if( mem.alarmCallback!=0
&& mem.nowUsed+mem.nowUsedMMap+nBytes>=mem.alarmThreshold ){
sqlite3MemsysAlarm(nBytes);
}
if( (p = mmapAlloc(nBytes))==0 ){
p = malloc(nBytes+8);
if( p==0 ){
sqlite3MemsysAlarm(nBytes);
p = malloc(nBytes+8);
}
if( p ){
p[0] = nBytes;
p++;
mem.nowUsed += nBytes;
if( mem.nowUsed>mem.mxUsed ){
mem.mxUsed = mem.nowUsed;
}
}
}
return (void*)p;
}
/*
** Return the size of a memory allocation
*/
static int memsys4Size(void *pPrior){
char *z = (char*)pPrior;
int idx = mem.nPage ? (z - mem.mmapHeap)/mem.szPage : 0;
int nByte;
if( idx>=1 && idx<mem.nPage ){
nByte = mem.szPage;
}else{
sqlite3_int64 *p = pPrior;
p--;
nByte = (int)*p;
}
return nByte;
}
/*
** Free memory.
*/
static void memsys4Free(void *pPrior){
sqlite3_int64 *p;
int nByte;
if( mmapFree(pPrior)==0 ){
p = pPrior;
p--;
nByte = (int)*p;
mem.nowUsed -= nByte;
free(p);
if( mem.nowUsed==0 ){
mmapUnmap();
}
}
}
/*
** Allocate nBytes of memory
*/
void *sqlite3_malloc(int nBytes){
sqlite3_int64 *p = 0;
if( nBytes>0 ){
memsys4Enter();
p = memsys4Malloc(nBytes);
sqlite3_mutex_leave(mem.mutex);
}
return (void*)p;
}
/*
** Free memory.
*/
void sqlite3_free(void *pPrior){
if( pPrior==0 ){
return;
}
assert( mem.mutex!=0 );
sqlite3_mutex_enter(mem.mutex);
memsys4Free(pPrior);
sqlite3_mutex_leave(mem.mutex);
}
/*
** Change the size of an existing memory allocation
*/
void *sqlite3_realloc(void *pPrior, int nBytes){
int nOld;
sqlite3_int64 *p;
if( pPrior==0 ){
return sqlite3_malloc(nBytes);
}
if( nBytes<=0 ){
sqlite3_free(pPrior);
return 0;
}
nOld = memsys4Size(pPrior);
if( nBytes<=nOld && nBytes>=nOld-128 ){
return pPrior;
}
assert( mem.mutex!=0 );
sqlite3_mutex_enter(mem.mutex);
p = memsys4Malloc(nBytes);
if( p ){
if( nOld<nBytes ){
memcpy(p, pPrior, nOld);
}else{
memcpy(p, pPrior, nBytes);
}
memsys4Free(pPrior);
}
assert( mem.mutex!=0 );
sqlite3_mutex_leave(mem.mutex);
return (void*)p;
}
#endif /* !SQLITE_MEMDEBUG && !SQLITE_OMIT_MEMORY_ALLOCATION */