From ff4917e9a537deeb0bdbbec7c9ea3849e8a095c6 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 19 Nov 2022 02:32:26 +0000 Subject: [PATCH 01/72] Create new branch named "base_convert" FossilOrigin-Name: 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 1fd11cbf21..dabd88714c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\soptimization\sin\sbtree.c. -D 2022-11-19T00:22:12.991 +C Create\snew\sbranch\snamed\s"base_convert" +D 2022-11-19T02:32:26.249 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2055,8 +2055,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff494449efd475878c549728cc22ee9b12d13674068781747fc042a0c1bd09c8 +P f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff R 8e4332decceeea7e1973b8e5ac88e8a4 -U drh -Z 80da1bc9f69008afc02dd88003bee918 +T *branch * base_convert +T *sym-base_convert * +T -sym-trunk * +U larrybr +Z 21ba949f5c7ccf39d76b47dd8fe1f905 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e33296fde..bc34d0feed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff \ No newline at end of file +0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea \ No newline at end of file From fb2e0bfa8b540fb6e7be5a74beaadd06b60570c1 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 19 Nov 2022 02:39:16 +0000 Subject: [PATCH 02/72] New extensions for base85 and base64 conversion UDFs FossilOrigin-Name: 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 --- ext/misc/base64.c | 233 +++++++++++++++++++++++++++++ ext/misc/base85.c | 366 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 15 +- manifest.uuid | 2 +- 4 files changed, 607 insertions(+), 9 deletions(-) create mode 100644 ext/misc/base64.c create mode 100644 ext/misc/base85.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c new file mode 100644 index 0000000000..1ed1a4f174 --- /dev/null +++ b/ext/misc/base64.c @@ -0,0 +1,233 @@ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane ASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes being the same as ASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that. +** But the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ + +typedef unsigned char ubyte; + +static const ubyte b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) +#define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[dv]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define DARK_MAX 72 + +/* Encode a byte buffer into base64 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + *pOut = 0; + while( nbIn > 0 ){ + static signed char ncio[] = { 0, 2, 3, 4 }; + int nbi = (nbIn > 3)? 3 : nbIn; + signed char nc; + int nbe; + unsigned long qv = (ubyte)*pIn++; + for( nbe=1; nbe<3; ++nbe ){ + ubyte b = (nbe=0; --nbe ){ + char ce = (nbe0 && *pIn!='=' ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + if( ncIn<=0 ) break; + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + nbo = nboi[nti]; + while( nti>0 ){ + char c = *pIn++; + ubyte bdp = BX_DV_PROTO(c); + --ncIn; + switch( bdp ){ + case WS: + case ND: + nti = 0; + break; + case PC: + bdp = 0; + /* fall thru */ + default: /* It's the digit value. */ + qv = qv<<6 | bdp; + --nti; + break; + } + } + while( nbo-- > 0 ){ + *pOut++ = (qv >> (8*nbo))&0xff; + } + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + ubyte *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big.", -1); + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (ubyte*)sqlite3_value_blob(av[0]); + nc = (int)(toBase64(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big.", -1); + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text.", -1); + break; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} diff --git a/ext/misc/base85.c b/ext/misc/base85.c new file mode 100644 index 0000000000..365db80f2b --- /dev/null +++ b/ext/misc/base85.c @@ -0,0 +1,366 @@ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane ASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit ASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +/* Classify c according to interval within ASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static unsigned char b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static unsigned char base85DigitValue( char c ){ + unsigned char dv = (unsigned char)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +static char * skipNonB85( char *s ){ + char c; + while( (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +static char base85Numeral( unsigned char b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} + +static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + *pOut = 0; + while( nbIn > 0 ){ + static signed char ncio[] = { 0, 2, 3, 4, 5 }; + int nbi = (nbIn > 4)? 4 : nbIn; + unsigned long qv = 0L; + int nbe = 0; + signed char nco; + while( nbe++ < nbi ){ + qv = (qv<<8) | *pIn++; + } + nco = ncio[nbi]; + nbIn -= nbi; + while( nco > 0 ){ + unsigned char dv = (unsigned char)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += ncio[nbi]; + if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + char *p = pSep; + while( *p ) *pOut++ = *p++; + nCol = 0; + } + *pOut = 0; + } + return pOut; +} + +static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + if( ncIn==0 ) break; + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + while( nti>0 ){ + char c = *pIn++; + unsigned char cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + c - cdo; + --nti; + } + nbo -= nti; + while( nbo-- > 0 ){ + *pOut++ = (qv >> (8*nbo))&0xff; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL.", -1); + break; + } +} +# endif + +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + unsigned char *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big.", -1); + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (unsigned char*)sqlite3_value_blob(av[0]); + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big.", -1); + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + break; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + unsigned char bBuf[64]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif diff --git a/manifest b/manifest index dabd88714c..faf2f3f49e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\snew\sbranch\snamed\s"base_convert" -D 2022-11-19T02:32:26.249 +C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs +D 2022-11-19T02:39:16.296 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,6 +289,8 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 +F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 +F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -2055,11 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff -R 8e4332decceeea7e1973b8e5ac88e8a4 -T *branch * base_convert -T *sym-base_convert * -T -sym-trunk * +P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea +R 4ecad003b3a6f60dd6b11d354f8edbe2 U larrybr -Z 21ba949f5c7ccf39d76b47dd8fe1f905 +Z d182470ca73df70df09238ac5a5e4382 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bc34d0feed..b47c8a0962 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea \ No newline at end of file +5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file From 423003dca91a84409950bfd4e99ee68dc4f831ee Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 21 Nov 2022 00:11:09 +0000 Subject: [PATCH 03/72] Fix a base64 decode bug. Provide for convenient inclusion of extension(s) built into the CLI, to simplify testing and for its own sake. Improve comments. Cure collision between base64.c and base85.c when both are in the same translation unit. FossilOrigin-Name: 07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f --- ext/misc/base64.c | 55 ++++++++++++++++++++++++++-------------- ext/misc/base85.c | 50 ++++++++++++++++++++++++++++++------ ext/misc/basexx.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++ manifest | 17 +++++++------ manifest.uuid | 2 +- src/shell.c.in | 33 ++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 ext/misc/basexx.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 1ed1a4f174..d6f7834a85 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -12,7 +12,7 @@ ** ** This is a SQLite extension for converting in either direction ** between a (binary) blob and base64 text. Base64 can transit a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals, ** and can be used unmodified in XML-like documents. ** @@ -37,10 +37,10 @@ ** error will be thrown for other input argument types. ** ** This code relies on UTF-8 encoding only with respect to the -** meaning of the first 128 (7-bit) codes being the same as ASCII. +** meaning of the first 128 (7-bit) codes matching that of USASCII. ** It will fail miserably if somehow made to try to convert EBCDIC. -** Because it is table-driven, it could be enhanced to handle that. -** But the world and SQLite have moved on from that anachronism. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. ** ** To build the extension: ** Set shell variable SQDIR= @@ -52,12 +52,16 @@ #include +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ #include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #define PC 0x80 /* pad character */ #define WS 0x81 /* whitespace */ #define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' typedef unsigned char ubyte; @@ -89,10 +93,10 @@ static const char b64Numerals[64] #define IS_BX_PAD(bdp) ((bdp)==PC) #define BX_NUMERAL(dv) (b64Numerals[dv]) /* Width of base64 lines. Should be an integer multiple of 4. */ -#define DARK_MAX 72 +#define B64_DARK_MAX 72 /* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to DARK_MAX +** to be appended to encoded groups to limit their length to B64_DARK_MAX ** or to terminate the last group (to aid concatenation.) */ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ @@ -111,12 +115,12 @@ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ nc = ncio[nbi]; nbIn -= nbi; for( nbe=3; nbe>=0; --nbe ){ - char ce = (nbe0 && *pIn!='=' ){ + while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; char *pUse = skipNonB64(pIn); unsigned long qv = 0L; - int nti, nbo; + int nti, nbo, nac; ncIn -= (pUse - pIn); - if( ncIn<=0 ) break; pIn = pUse; nti = (ncIn>4)? 4 : ncIn; nbo = nboi[nti]; - while( nti>0 ){ - char c = *pIn++; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac<=nti)? *pIn++ : PAD_CHAR; ubyte bdp = BX_DV_PROTO(c); --ncIn; switch( bdp ){ @@ -157,15 +161,16 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; case PC: bdp = 0; + --nbo; /* fall thru */ - default: /* It's the digit value. */ + default: /* bdp is the digit value. */ qv = qv<<6 | bdp; - --nti; break; } } + nti = 2; while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nbo))&0xff; + *pOut++ = (qv >> (8*nti--))&0xff; } } return pOut; @@ -183,7 +188,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ case SQLITE_BLOB: nb = nv; nc = 4*(nv+2/3); /* quads needed */ - nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big.", -1); } @@ -219,11 +224,15 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ /* ** Establish linkage to running SQLite library. */ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; return sqlite3_create_function @@ -231,3 +240,11 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, 0, base64, 0, 0); } + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 365db80f2b..dfa9bd0e31 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -14,7 +14,7 @@ ** It can be built as a standalone program or an SQLite3 extension. ** ** Much like base64 representations, base85 can be sent through a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals. ** It is not suited for unmodified use in XML-like documents. ** @@ -24,7 +24,7 @@ ** Further, this is an independent implementation of a base85 system. ** Hence, the author has rightfully put this into the public domain. ** -** Base85 numerals are taken from the set of 7-bit ASCII codes, +** Base85 numerals are taken from the set of 7-bit USASCII codes, ** excluding control characters and Space ! " ' ( ) { | } ~ Del ** in code order representing digit values 0 to 84 (base 10.) ** @@ -32,8 +32,11 @@ ** are represented as 5-digit base85 numbers with MS to LS digit ** order. Groups of 1-3 bytes are represented with 2-4 digits, ** still big-endian but 8-24 bit values. (Using big-endian yields -** the simplest transition to byte groups smaller than 4 bytes.) +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) ** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. ** ** Any character not in the base85 numeral set delimits groups. ** When base85 is streamed or stored in containers of indefinite @@ -82,7 +85,10 @@ #ifndef BASE85_STANDALONE +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ # include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #else @@ -107,7 +113,7 @@ static void sayHelp(){ } #endif -/* Classify c according to interval within ASCII set w.r.t. base85 +/* Classify c according to interval within USASCII set w.r.t. base85 * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. */ #define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) @@ -127,16 +133,25 @@ static unsigned char base85DigitValue( char c ){ } #endif +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + static char * skipNonB85( char *s ){ char c; while( (c = *s) && !IS_B85(c) ) ++s; return s; } +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/ static char base85Numeral( unsigned char b ){ return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); } +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ int nCol = 0; *pOut = 0; @@ -157,7 +172,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ pOut[--nco] = base85Numeral(dv); } pOut += ncio[nbi]; - if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + if( pSep && ((nCol += ncio[nbi])>=B85_DARK_MAX || nbIn<=0) ){ char *p = pSep; while( *p ) *pOut++ = *p++; nCol = 0; @@ -167,6 +182,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ return pOut; } +/* Decode base85 text into a byte buffer. */ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ @@ -196,6 +212,7 @@ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ } #ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ static int allBase85( char *p, int len ){ char c; while( len-- > 0 && (c = *p++) != 0 ){ @@ -208,6 +225,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE # ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); switch( sqlite3_value_type(av[0]) ){ @@ -228,6 +246,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ } # endif +/* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ int nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), @@ -272,11 +291,18 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "base85 OOM", -1); } +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; # ifndef OMIT_BASE85_CHECKER @@ -294,12 +320,20 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, 0, base85, 0, 0); } +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + #else /* standalone program */ int main(int na, char *av[]){ int cin; int rc = 0; - unsigned char bBuf[64]; + unsigned char bBuf[4*(B85_DARK_MAX/5)]; char cBuf[5*(sizeof(bBuf)/4)+2]; size_t nio; # ifndef OMIT_BASE85_CHECKER diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c new file mode 100644 index 0000000000..700a99409c --- /dev/null +++ b/ext/misc/basexx.c @@ -0,0 +1,64 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + init_api_ptr(pApi); + int rc1 = BASE64_INIT(db); + int rc2 = BASE85_INIT(db); + int rc = SQLITE_OK; + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/manifest b/manifest index faf2f3f49e..f02b4834b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs -D 2022-11-19T02:39:16.296 +C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. +D 2022-11-21T00:11:09.323 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,8 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 -F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 +F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 +F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -640,7 +641,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 49b6aeaf950b9b7a74aca7b80e6d004ed32f0a16d2c9471e34587cc4755d5eaa F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea -R 4ecad003b3a6f60dd6b11d354f8edbe2 +P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 +R 0260a27c5340f9b412a0f6e0981a0d6d U larrybr -Z d182470ca73df70df09238ac5a5e4382 +Z 18487c3a1bcdc27a89418b38ad3e3f97 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b47c8a0962..41284b2a7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file +07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..478c055357 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1070,6 +1070,9 @@ INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h INCLUDE ../ext/recover/sqlite3recover.c #endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) +#endif #if defined(SQLITE_ENABLE_SESSION) /* @@ -5115,6 +5118,7 @@ static void open_db(ShellState *p, int openFlags){ } exit(1); } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif @@ -5137,6 +5141,34 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_sqlar_init(p->db, 0, 0); } #endif +#ifdef SQLITE_SHELL_EXTFUNCS + /* Create a preprocessing mechanism for extensions to make + * their own provisions for being built into the shell. + * This is a short-span macro. See further below for usage. + */ +#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant +#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant) + /* Let custom-included extensions get their ..._init() called. + * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause + * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) + * inititialization routine to be called. + */ + { + int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); + /* Let custom-included extensions expose their functionality. + * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause + * the SQL functions, virtual tables, collating sequences or + * VFS's implemented by the extension to be registered. + */ + if( irc==SQLITE_OK + || irc==SQLITE_OK_LOAD_PERMANENTLY ){ + SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0); + } +#undef SHELL_SUB_MACRO +#undef SHELL_SUBMACRO + } +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, @@ -5157,6 +5189,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); #endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename); From b5aaf0f28507a27a2ea8e18f642625d8727e9c48 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 21 Nov 2022 02:18:12 +0000 Subject: [PATCH 04/72] Get clean builds for MS tools. Add build examples for extension glommer (basexx.c) Make sure extensions build for dynamic load and built into shell. FossilOrigin-Name: da940d507e9e31a84daedede44a56f743198d7258a9e83892511fe8dd4c78d2b --- ext/misc/base64.c | 2 +- ext/misc/basexx.c | 22 +++++++++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/ext/misc/base64.c b/ext/misc/base64.c index d6f7834a85..1ab0dc4ead 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -84,7 +84,7 @@ static const ubyte b64DigitValues[128] = { 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND }; -static const char b64Numerals[64] +static const char b64Numerals[64+1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; #define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c index 700a99409c..a113134c85 100644 --- a/ext/misc/basexx.c +++ b/ext/misc/basexx.c @@ -16,6 +16,23 @@ ** ** Presently, it combines the base64.c and base85.c extensions. However, ** it can be used as a template for other combinations. +** +** Example usages: +** +** - Build a runtime-loadable extension from SQLite checkout directory: +** *Nix, OSX: gcc -O2 -shared -I. -fPIC -o basexx.so ext/misc/basexx.c +** Win32: cl /Os -I. ext/misc/basexx.c -link -dll -out:basexx.dll +** +** - Incorporate as built-in in sqlite3 shell: +** *Nix, OSX with gcc on a like platform: +** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** make sqlite3 "OPTS=$mop1 $mop2" +** Win32 with Microsoft toolset on Windows: +** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** set mops="OPTS=%mop1% %mop2%" +** nmake -f Makefile.msc sqlite3.exe %mops% */ #ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ @@ -44,7 +61,10 @@ static void init_api_ptr(const sqlite3_api_routines *pApi){ #define sqlite3_base_init sqlite3_base85_init #include "base85.c" -static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_basexx_init(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ init_api_ptr(pApi); int rc1 = BASE64_INIT(db); diff --git a/manifest b/manifest index f02b4834b7..6d28b2a340 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. -D 2022-11-21T00:11:09.323 +C Get\sclean\sbuilds\sfor\sMS\stools.\sAdd\sbuild\sexamples\sfor\sextension\sglommer\s(basexx.c)\nMake\ssure\sextensions\sbuild\sfor\sdynamic\sload\sand\sbuilt\sinto\sshell. +D 2022-11-21T02:18:12.614 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,9 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base64.c e611e8ce552b16251c1691191d95912f9e03400637311dbd458062d2d0b95e2a F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 -F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 +F ext/misc/basexx.c 678dcc83894f78c26fd3662b322886777cc26bf2b40809236cd2abdad532a33c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 -R 0260a27c5340f9b412a0f6e0981a0d6d +P 07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f +R 1b52ba3002b9af5fda34f27079570bff U larrybr -Z 18487c3a1bcdc27a89418b38ad3e3f97 +Z 7a2c313c3d99c0b1eaaafc6ea380893a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 41284b2a7c..81f4745948 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file +da940d507e9e31a84daedede44a56f743198d7258a9e83892511fe8dd4c78d2b \ No newline at end of file From 03af6d7157c0053fad2a55286004e0f16ea6e95f Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Nov 2022 16:40:12 +0000 Subject: [PATCH 05/72] Change the name of the Parse.pIdxExpr field to pIdxEpr so that the name is distinct from all other identifiers in the code and thus less confusing. FossilOrigin-Name: a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 12 ++++++------ src/sqliteInt.h | 2 +- src/where.c | 22 +++++++++++----------- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index ab782561db..d3ee673914 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\scontention\stest:\sgive\seach\sworker\sa\sdistinct\srecognizable\sname\sinstead\sof\sa\srandom\sone. -D 2022-11-21T16:00:26.047 +C Change\sthe\sname\sof\sthe\sParse.pIdxExpr\sfield\sto\spIdxEpr\sso\sthat\sthe\sname\sis\ndistinct\sfrom\sall\sother\sidentifiers\sin\sthe\scode\sand\sthus\sless\sconfusing. +D 2022-11-21T16:40:12.023 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 847f87d9df3ede2b2b0a8db088af0b9c1923b21009f8ea1b9b7b28cb0a383170 +F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -646,7 +646,7 @@ F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 1a2cec0f7682da69975eb203fef2058949051ff500445b997d8047cbd8813315 +F src/sqliteInt.h 42bcbb6cebef64c99068e7bc141a2541d40d65c7c26a9ca5ada4d83ca353555d F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 1ef5aae7fac877057b9f360f06b26d4275888460d8fb6e92bbb9e70e07afe946 +F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bee94d1bb0daade023cc1e274339daafc249e1978c0765fc45042b5f9060e478 -R 24e9b381879a473eda77c6d52873b2a6 -U stephan -Z 5200ed97b9b65f90e5d50e34624e223d +P 5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 +R fadcb96891525abc2e619c860d5616b7 +U drh +Z 75407774f59d7d22e403811dc4acdbf3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2dbda6c4cc..987c32b82f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 \ No newline at end of file +a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 7a4e59f28d..fb718eb23e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4037,7 +4037,7 @@ static int exprCodeInlineFunction( } /* -** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr. +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. ** If it is, then resolve the expression by reading from the index and ** return the register into which the value has been read. If pExpr is ** not an indexed expression, then return negative. @@ -4049,7 +4049,7 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( ){ IndexedExpr *p; Vdbe *v; - for(p=pParse->pIdxExpr; p; p=p->pIENext){ + for(p=pParse->pIdxEpr; p; p=p->pIENext){ int iDataCur = p->iDataCur; if( iDataCur<0 ) continue; if( pParse->iSelfTab ){ @@ -4069,10 +4069,10 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); sqlite3VdbeGoto(v, 0); - p = pParse->pIdxExpr; - pParse->pIdxExpr = 0; + p = pParse->pIdxEpr; + pParse->pIdxEpr = 0; sqlite3ExprCode(pParse, pExpr, target); - pParse->pIdxExpr = p; + pParse->pIdxEpr = p; sqlite3VdbeJumpHere(v, addr+2); }else{ sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); @@ -4111,7 +4111,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ expr_code_doover: if( pExpr==0 ){ op = TK_NULL; - }else if( pParse->pIdxExpr!=0 + }else if( pParse->pIdxEpr!=0 && !ExprHasProperty(pExpr, EP_Leaf) && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 469ce9bba1..6ff49ab5d0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3656,7 +3656,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ - IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */ + IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ diff --git a/src/where.c b/src/where.c index 7ba72d2766..ae3d513cae 100644 --- a/src/where.c +++ b/src/where.c @@ -4591,8 +4591,8 @@ static i8 wherePathSatisfiesOrderBy( if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; }else{ - Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr; - if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){ + Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr; + if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){ continue; } } @@ -5461,13 +5461,13 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( /* ** This is an sqlite3ParserAddCleanup() callback that is invoked to -** free the Parse->pIdxExpr list when the Parse object is destroyed. +** free the Parse->pIdxEpr list when the Parse object is destroyed. */ static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ Parse *pParse = (Parse*)pObject; - while( pParse->pIdxExpr!=0 ){ - IndexedExpr *p = pParse->pIdxExpr; - pParse->pIdxExpr = p->pIENext; + while( pParse->pIdxEpr!=0 ){ + IndexedExpr *p = pParse->pIdxEpr; + pParse->pIdxEpr = p->pIENext; sqlite3ExprDelete(db, p->pExpr); sqlite3DbFreeNN(db, p); } @@ -5479,13 +5479,13 @@ static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ ** number for the index and iDataCur is the cursor number for the corresponding ** table. ** -** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for +** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for ** each of the expressions in the index so that the expression code generator ** will know to replace occurrences of the indexed expression with ** references to the corresponding column of the index. */ static SQLITE_NOINLINE void whereAddIndexedExpr( - Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */ Index *pIdx, /* The index-on-expression that contains the expressions */ int iIdxCur, /* Cursor number for pIdx */ SrcItem *pTabItem /* The FROM clause entry for the table */ @@ -5514,7 +5514,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( if( sqlite3ExprIsConstant(pExpr) ) continue; p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; - p->pIENext = pParse->pIdxExpr; + p->pIENext = pParse->pIdxEpr; p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; @@ -5523,7 +5523,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS p->zIdxName = pIdx->zName; #endif - pParse->pIdxExpr = p; + pParse->pIdxEpr = p; if( p->pIENext==0 ){ sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); } @@ -6474,7 +6474,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ last = pWInfo->iEndWhere; } if( pIdx->bHasExpr ){ - IndexedExpr *p = pParse->pIdxExpr; + IndexedExpr *p = pParse->pIdxEpr; while( p ){ if( p->iIdxCur==pLevel->iIdxCur ){ p->iDataCur = -1; From da217c958eb14824c3bd308a279d9cad19fea35b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Nov 2022 17:40:23 +0000 Subject: [PATCH 06/72] Split out the debugging code that dumps an AggInfo object into a separate subroutine called printAggInfo() so that it can be invoked interactively during debugging. No changes to production code. FossilOrigin-Name: dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 41 +++++++++++++++++++++++++---------------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index d3ee673914..d6216cacf6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sParse.pIdxExpr\sfield\sto\spIdxEpr\sso\sthat\sthe\sname\sis\ndistinct\sfrom\sall\sother\sidentifiers\sin\sthe\scode\sand\sthus\sless\sconfusing. -D 2022-11-21T16:40:12.023 +C Split\sout\sthe\sdebugging\scode\sthat\sdumps\san\sAggInfo\sobject\sinto\sa\sseparate\nsubroutine\scalled\sprintAggInfo()\sso\sthat\sit\scan\sbe\sinvoked\sinteractively\nduring\sdebugging.\s\sNo\schanges\sto\sproduction\scode. +D 2022-11-21T17:40:23.307 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 +F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 -R fadcb96891525abc2e619c860d5616b7 +P a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe +R 9fdeab1b213168256e8eff07917f0804 U drh -Z 75407774f59d7d22e403811dc4acdbf3 +Z 716f42c6984876d9d59330e78a2ea955 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 987c32b82f..f17714d1e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe \ No newline at end of file +dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b0f79f840b..f996ac0852 100644 --- a/src/select.c +++ b/src/select.c @@ -6629,6 +6629,30 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ return 0; } +#if TREETRACE_ENABLED +/* +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pCol->iMem, + pCol->iSorterColumn); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->aFunc[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7428,28 +7452,13 @@ int sqlite3Select( if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x400 ){ - int ii; SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); if( minMaxFlag ){ sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); } - for(ii=0; iinColumn; ii++){ - struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; - sqlite3DebugPrintf( - "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" - " iSorterColumn=%d\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, pCol->iMem, - pCol->iSorterColumn); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } + printAggInfo(pAggInfo); } #endif From c59b7a8053f0bad8d5df2d834b446a0594ade49c Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 12:47:32 +0000 Subject: [PATCH 07/72] Convert an ALWAYS() in sqlite3DbSpanDup() into an assert(), for a performance increase and size reduction. FossilOrigin-Name: 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d6216cacf6..f5beaaa03b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Split\sout\sthe\sdebugging\scode\sthat\sdumps\san\sAggInfo\sobject\sinto\sa\sseparate\nsubroutine\scalled\sprintAggInfo()\sso\sthat\sit\scan\sbe\sinvoked\sinteractively\nduring\sdebugging.\s\sNo\schanges\sto\sproduction\scode. -D 2022-11-21T17:40:23.307 +C Convert\san\sALWAYS()\sin\ssqlite3DbSpanDup()\sinto\san\sassert(),\sfor\sa\sperformance\nincrease\sand\ssize\sreduction. +D 2022-11-22T12:47:32.742 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -605,7 +605,7 @@ F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9 F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe -F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90 +F src/malloc.c 3d4ec162214024ee071d85711b93bec25cd3371280aee3702b63bcf312ca8238 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe -R 9fdeab1b213168256e8eff07917f0804 +P dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 +R 6f72c126be40c707db8479e3d23315a3 U drh -Z 716f42c6984876d9d59330e78a2ea955 +Z 9806e837fcd3094e05cfafc89f1e72fe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f17714d1e3..02e6c3a494 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 \ No newline at end of file +21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index af83743fc0..68d9f5f55c 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -793,9 +793,14 @@ char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ */ char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ int n; +#ifdef SQLITE_DEBUG + /* Because of the way the parser works, the span is guaranteed to contain + ** at least one non-space character */ + for(n=0; sqlite3Isspace(zStart[n]); n++){ assert( &zStart[n]0) && sqlite3Isspace(zStart[n-1]) ) n--; + while( sqlite3Isspace(zStart[n-1]) ) n--; return sqlite3DbStrNDup(db, zStart, n); } From ff5e4eccedbfc2294a8072888de596a8825adee0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 13:33:58 +0000 Subject: [PATCH 08/72] This is the first in what is anticipated to be a long sequence of incremental changes aimed at improving aggregate query processing, and in particular helping aggregate queries take better advantage of indexes on expression. The end goal is to resolve ticket [99378177930f87bd], though it remains to be seen whether or not I can get there with this approach. FossilOrigin-Name: cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/select.c | 10 ++++------ src/sqliteInt.h | 2 +- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f5beaaa03b..bf95667970 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\san\sALWAYS()\sin\ssqlite3DbSpanDup()\sinto\san\sassert(),\sfor\sa\sperformance\nincrease\sand\ssize\sreduction. -D 2022-11-22T12:47:32.742 +C This\sis\sthe\sfirst\sin\swhat\sis\santicipated\sto\sbe\sa\slong\ssequence\sof\sincremental\nchanges\saimed\sat\simproving\saggregate\squery\sprocessing,\sand\sin\sparticular\nhelping\saggregate\squeries\stake\sbetter\sadvantage\sof\sindexes\son\sexpression.\nThe\send\sgoal\sis\sto\sresolve\sticket\s[99378177930f87bd],\sthough\sit\sremains\sto\nbe\sseen\swhether\sor\snot\sI\scan\sget\sthere\swith\sthis\sapproach. +D 2022-11-22T13:33:58.528 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d +F src/select.c 0f78363cb997bba3df6f974c5022b1cef9c6d7bc9ae73bcfd75b1e4b5f7d95fc F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 42bcbb6cebef64c99068e7bc141a2541d40d65c7c26a9ca5ada4d83ca353555d +F src/sqliteInt.h 62ff4e899d1a10e7fe60617f9d58ad49f20de4ad30ee02b68dc403ece984b617 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 -R 6f72c126be40c707db8479e3d23315a3 +P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 +R 8a6d83ef799149939ad05ed6225df7c1 +T *branch * agg-with-indexed-expr +T *sym-agg-with-indexed-expr * +T -sym-trunk * U drh -Z 9806e837fcd3094e05cfafc89f1e72fe +Z c8ad55df8af287c23ae8c2fae9e521d0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02e6c3a494..3f8ba1bdb5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 \ No newline at end of file +cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 \ No newline at end of file diff --git a/src/select.c b/src/select.c index f996ac0852..5306eb399b 100644 --- a/src/select.c +++ b/src/select.c @@ -6223,18 +6223,17 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pParse->nErr ) return; #ifdef SQLITE_DEBUG /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); + ** AggInfo.mnReg..(AggInfo.mnReg+nReg-1) */ for(i=0; inColumn; i++){ assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); + && pAggInfo->aCol[i].iMemmnReg+nReg ); } for(i=0; inFunc; i++){ assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); + && pAggInfo->aFunc[i].iMemmnReg+nReg ); } #endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mnReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -7448,7 +7447,6 @@ int sqlite3Select( #endif sNC.ncFlags &= ~NC_InAggFunc; } - pAggInfo->mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x400 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6ff49ab5d0..d4ffaa2251 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2719,7 +2719,7 @@ struct AggInfo { int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int mnReg; /* First in a range of regsiters for aCol and aFunc */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ From 42b78237cdd11b79e4c37d0373a338c1409f8283 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 14:10:22 +0000 Subject: [PATCH 09/72] Factor out the allocation of registers for aggregates into a separate subroutine. FossilOrigin-Name: 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/select.c | 21 ++++++++++++++++++++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index bf95667970..7f4b62ade6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sis\sthe\sfirst\sin\swhat\sis\santicipated\sto\sbe\sa\slong\ssequence\sof\sincremental\nchanges\saimed\sat\simproving\saggregate\squery\sprocessing,\sand\sin\sparticular\nhelping\saggregate\squeries\stake\sbetter\sadvantage\sof\sindexes\son\sexpression.\nThe\send\sgoal\sis\sto\sresolve\sticket\s[99378177930f87bd],\sthough\sit\sremains\sto\nbe\sseen\swhether\sor\snot\sI\scan\sget\sthere\swith\sthis\sapproach. -D 2022-11-22T13:33:58.528 +C Factor\sout\sthe\sallocation\sof\sregisters\sfor\saggregates\sinto\sa\sseparate\nsubroutine. +D 2022-11-22T14:10:22.456 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9 +F src/expr.c 5de79bb844b02b6ebc6778161e549d1e3f2a8c0f203234d0212bd97161ed81bc F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 0f78363cb997bba3df6f974c5022b1cef9c6d7bc9ae73bcfd75b1e4b5f7d95fc +F src/select.c 32d4f232ee178a4a57595061f26b10d6c51c3530cc3e721311c586201f41337a F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,11 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 -R 8a6d83ef799149939ad05ed6225df7c1 -T *branch * agg-with-indexed-expr -T *sym-agg-with-indexed-expr * -T -sym-trunk * +P cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 +R 8ec59d9575fc141fe0aff25c195ea672 U drh -Z c8ad55df8af287c23ae8c2fae9e521d0 +Z dd7dd4863c6318a4016da919a253840a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f8ba1bdb5..d932c72af1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 \ No newline at end of file +4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fb718eb23e..f6b065208b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4441,6 +4441,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ + assert( pInfo->aFunc[pExpr->iAgg].iMem>0 ); return pInfo->aFunc[pExpr->iAgg].iMem; } break; @@ -4730,6 +4731,7 @@ expr_code_doover: if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ + assert( pAggInfo->aCol[pExpr->iAgg].iMem>0 ); inReg = pAggInfo->aCol[pExpr->iAgg].iMem; break; } @@ -6294,7 +6296,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pCol->pTab = pExpr->y.pTab; pCol->iTable = pExpr->iTable; pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; pCol->iSorterColumn = -1; pCol->pCExpr = pExpr; if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ @@ -6357,7 +6358,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; - pItem->iMem = ++pParse->nMem; assert( ExprUseUToken(pExpr) ); pItem->pFunc = sqlite3FindFunction(pParse->db, pExpr->u.zToken, diff --git a/src/select.c b/src/select.c index 5306eb399b..0872a95454 100644 --- a/src/select.c +++ b/src/select.c @@ -6204,6 +6204,21 @@ void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +/* +** Assign register numbers to all pAggInfo->aCol[] and pAggInfo->aFunc[] +** entries. +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + int i, m; + assert( pAggInfo!=0 ); + assert( pAggInfo->mnReg==0 ); + m = pParse->nMem; + pAggInfo->mnReg = m+1; + for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; + for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; + pParse->nMem = m; +} + /* ** Reset the aggregate accumulator. ** @@ -6219,6 +6234,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); + assert( pAggInfo->mnReg>0 ); /* assignAggregateRegisters() has been run */ if( nReg==0 ) return; if( pParse->nErr ) return; #ifdef SQLITE_DEBUG @@ -7413,7 +7429,6 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - pAggInfo->mnReg = pParse->nMem+1; pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); @@ -7534,6 +7549,7 @@ int sqlite3Select( sqlite3ExprListDelete(db, pDistinct); goto select_end; } + assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ @@ -7772,6 +7788,8 @@ int sqlite3Select( if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } + assignAggregateRegisters(pParse, pAggInfo); + assert( pAggInfo->aFunc[0].iMem>=0 ); sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); @@ -7808,6 +7826,7 @@ int sqlite3Select( pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } + assignAggregateRegisters(pParse, pAggInfo); /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row From 89e5dfac01904ef55be9f2e5c007b25d57bc0263 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 14:31:13 +0000 Subject: [PATCH 10/72] Omit the unnecessary AggInfo.mnReg field. FossilOrigin-Name: d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 26 ++++++++++++++++---------- src/sqliteInt.h | 1 - 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 7f4b62ade6..e95765ddfb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\sthe\sallocation\sof\sregisters\sfor\saggregates\sinto\sa\sseparate\nsubroutine. -D 2022-11-22T14:10:22.456 +C Omit\sthe\sunnecessary\sAggInfo.mnReg\sfield. +D 2022-11-22T14:31:13.165 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 32d4f232ee178a4a57595061f26b10d6c51c3530cc3e721311c586201f41337a +F src/select.c 27822fc0fe69d316448294c44c638bd1aad9b980ad62f0a7bb501555245eeb26 F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 62ff4e899d1a10e7fe60617f9d58ad49f20de4ad30ee02b68dc403ece984b617 +F src/sqliteInt.h a16a36f7b2a77b66f8ed0a460aad9f0a5f2897026a51d656cb3556461d1f6be3 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 -R 8ec59d9575fc141fe0aff25c195ea672 +P 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d +R 5936ef480bfa81b2caa13a6a172023d0 U drh -Z dd7dd4863c6318a4016da919a253840a +Z 8bcded315dd8c6f4a10b877ff07bc1b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d932c72af1..8897d99aa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d \ No newline at end of file +d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0872a95454..dd02265958 100644 --- a/src/select.c +++ b/src/select.c @@ -6211,9 +6211,7 @@ void sqlite3SelectPrep( static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ int i, m; assert( pAggInfo!=0 ); - assert( pAggInfo->mnReg==0 ); m = pParse->nMem; - pAggInfo->mnReg = m+1; for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; pParse->nMem = m; @@ -6230,26 +6228,34 @@ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; + int iFirstReg; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); - assert( pAggInfo->mnReg>0 ); /* assignAggregateRegisters() has been run */ if( nReg==0 ) return; if( pParse->nErr ) return; + if( pAggInfo->nColumn==0 ){ + iFirstReg = pAggInfo->aFunc[0].iMem; + }else{ + iFirstReg = pAggInfo->aCol[0].iMem; + } #ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..(AggInfo.mnReg+nReg-1) */ + /* Verify that all AggInfo register numbers have been assigned and that + ** they are all sequential. */ + assert( iFirstReg>0 ); for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMemmnReg+nReg ); + assert( pAggInfo->aCol[i].iMem>=iFirstReg ); + assert( i==0 || pAggInfo->aCol[i].iMem==pAggInfo->aCol[i-1].iMem+1 ); } for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMemmnReg+nReg ); + assert( pAggInfo->aFunc[i].iMem>=iFirstReg ); + assert( i>0 || pAggInfo->nColumn==0 + || pAggInfo->aFunc[i].iMem==pAggInfo->aCol[pAggInfo->nColumn-1].iMem+1 ); + assert( i==0 || pAggInfo->aFunc[i].iMem==pAggInfo->aFunc[i-1].iMem+1 ); } #endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mnReg+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, iFirstReg, iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d4ffaa2251..68dbb3ab9f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2719,7 +2719,6 @@ struct AggInfo { int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg; /* First in a range of regsiters for aCol and aFunc */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ From 2a072506ca4ca14839f88ca1c027436e9d68c7c4 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 22 Nov 2022 15:41:32 +0000 Subject: [PATCH 11/72] Fix last function pointer in sqlite3Apis init. Reported at https://sqlite.org/forum/info/eba0faa96d FossilOrigin-Name: 6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/loadext.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f5beaaa03b..e1376b3753 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\san\sALWAYS()\sin\ssqlite3DbSpanDup()\sinto\san\sassert(),\sfor\sa\sperformance\nincrease\sand\ssize\sreduction. -D 2022-11-22T12:47:32.742 +C Fix\slast\sfunction\spointer\sin\ssqlite3Apis\sinit.\sReported\sat\shttps://sqlite.org/forum/info/eba0faa96d +D 2022-11-22T15:41:32.493 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -603,7 +603,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9 +F src/loadext.c 25663175950c5c4404b9377840b7b4c6fe5c53b415caf43634c62f442c02a9a7 F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe F src/malloc.c 3d4ec162214024ee071d85711b93bec25cd3371280aee3702b63bcf312ca8238 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 -R 6f72c126be40c707db8479e3d23315a3 -U drh -Z 9806e837fcd3094e05cfafc89f1e72fe +P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 +R b2f32015706aa1808841626a0ff1391c +U larrybr +Z d9d2dcaeb03b2f8d48cf12bc3dcb589d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02e6c3a494..1264b5622a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 \ No newline at end of file +6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 421ccca804..c14338f8b4 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -510,7 +510,7 @@ static const sqlite3_api_routines sqlite3Apis = { #endif sqlite3_db_name, /* Version 3.40.0 and later */ - sqlite3_value_type + sqlite3_value_encoding }; /* True if x is the directory separator character From 3c8e438583c617eea850ff52bec56f5db5a8d099 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 15:43:16 +0000 Subject: [PATCH 12/72] Since the memory registers used by the columns and functions of an AggInfo object are sequential, it is not neecessary to remember each register separately. We can simply remember the first one and do the math when others are needed. FossilOrigin-Name: dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 9 +++------ src/select.c | 49 ++++++++++++++----------------------------------- src/sqliteInt.h | 11 ++++++++--- 5 files changed, 34 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index e95765ddfb..8e8fab603a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\sunnecessary\sAggInfo.mnReg\sfield. -D 2022-11-22T14:31:13.165 +C Since\sthe\smemory\sregisters\sused\sby\sthe\scolumns\sand\sfunctions\sof\san\nAggInfo\sobject\sare\ssequential,\sit\sis\snot\sneecessary\sto\sremember\seach\sregister\nseparately.\s\sWe\scan\ssimply\sremember\sthe\sfirst\sone\sand\sdo\sthe\smath\swhen\nothers\sare\sneeded. +D 2022-11-22T15:43:16.425 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 5de79bb844b02b6ebc6778161e549d1e3f2a8c0f203234d0212bd97161ed81bc +F src/expr.c 63cce2c219748d8f8e00a30e4dc43d65b686b62489f154eebe892933bfb4a249 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 27822fc0fe69d316448294c44c638bd1aad9b980ad62f0a7bb501555245eeb26 +F src/select.c cf17743a695b3f4ff2dbd06d2ca8152997d37e3b3fd951efe0a67966740bd2d7 F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h a16a36f7b2a77b66f8ed0a460aad9f0a5f2897026a51d656cb3556461d1f6be3 +F src/sqliteInt.h fd605195929a3ce7f20da20b6a4da32045f9d4b343b51d94a0c7fdbad6207f24 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d -R 5936ef480bfa81b2caa13a6a172023d0 +P d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b +R a8a65a86f26eb6db314813261a6e0287 U drh -Z 8bcded315dd8c6f4a10b877ff07bc1b3 +Z 3374f8495add90924d12780ade06b13a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8897d99aa9..b43b5e65eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b \ No newline at end of file +dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f6b065208b..42099536ed 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4128,8 +4128,7 @@ expr_code_doover: assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - return pCol->iMem; + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); }else if( pAggInfo->useSortingIdx ){ Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, @@ -4441,8 +4440,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - assert( pInfo->aFunc[pExpr->iAgg].iMem>0 ); - return pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } @@ -4731,8 +4729,7 @@ expr_code_doover: if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ - assert( pAggInfo->aCol[pExpr->iAgg].iMem>0 ); - inReg = pAggInfo->aCol[pExpr->iAgg].iMem; + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); break; } if( pExpr->pAggInfo->useSortingIdx ){ diff --git a/src/select.c b/src/select.c index dd02265958..e40de38a59 100644 --- a/src/select.c +++ b/src/select.c @@ -6209,12 +6209,10 @@ void sqlite3SelectPrep( ** entries. */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ - int i, m; assert( pAggInfo!=0 ); - m = pParse->nMem; - for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; - for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; - pParse->nMem = m; + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; } /* @@ -6228,34 +6226,15 @@ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; - int iFirstReg; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; + assert( pAggInfo->iFirstReg>0 ); if( pParse->nErr ) return; - if( pAggInfo->nColumn==0 ){ - iFirstReg = pAggInfo->aFunc[0].iMem; - }else{ - iFirstReg = pAggInfo->aCol[0].iMem; - } -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo register numbers have been assigned and that - ** they are all sequential. */ - assert( iFirstReg>0 ); - for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=iFirstReg ); - assert( i==0 || pAggInfo->aCol[i].iMem==pAggInfo->aCol[i-1].iMem+1 ); - } - for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=iFirstReg ); - assert( i>0 || pAggInfo->nColumn==0 - || pAggInfo->aFunc[i].iMem==pAggInfo->aCol[pAggInfo->nColumn-1].iMem+1 ); - assert( i==0 || pAggInfo->aFunc[i].iMem==pAggInfo->aFunc[i-1].iMem+1 ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, iFirstReg, iFirstReg+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -6287,7 +6266,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; - sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } @@ -6375,7 +6355,7 @@ static void updateAccumulator( if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -6390,7 +6370,7 @@ static void updateAccumulator( addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); } pAggInfo->directMode = 0; @@ -6662,13 +6642,13 @@ static void printAggInfo(AggInfo *pAggInfo){ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d\n", ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, pCol->iMem, + pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), pCol->iSorterColumn); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); + sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + ii, AggInfoFuncReg(pAggInfo,ii)); sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); } } @@ -7795,8 +7775,7 @@ int sqlite3Select( sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } assignAggregateRegisters(pParse, pAggInfo); - assert( pAggInfo->aFunc[0].iMem>=0 ); - sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 68dbb3ab9f..45075cc361 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2716,15 +2716,15 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iMem; /* Memory location that acts as accumulator */ i16 iColumn; /* Column number within the source table */ i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; @@ -2735,7 +2735,6 @@ struct AggInfo { struct AggInfo_func { /* For each aggregate function */ Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ } *aFunc; @@ -2743,6 +2742,12 @@ struct AggInfo { u32 selId; /* Select to which this AggInfo belongs */ }; +/* +** Macros to compute aCol[] and aFunc[] register numbers: +*/ +#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) ((A)->iFirstReg+(A)->nColumn+(I)) + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater From 63b74e02a70b499d3783b1116e49840cb7a97e87 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 22 Nov 2022 16:12:53 +0000 Subject: [PATCH 13/72] Add Makefile.in targets for sqlite3r.c and sqlite3r.h, versions of the amalgamation that include the recover extension. FossilOrigin-Name: 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b --- Makefile.in | 9 +++++++++ manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/shell.c.in | 4 +++- tool/mksqlite3c.tcl | 18 ++++++++++++++---- tool/mksqlite3h.tcl | 10 ++++++++++ 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/Makefile.in b/Makefile.in index 09e75bb882..0a09e999f8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -778,6 +778,15 @@ sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . +sqlite3r.h: sqlite3.h + $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h + +sqlite3r.c: sqlite3.c sqlite3r.h + cp $(TOP)/ext/recover/sqlite3recover.c tsrc/ + cp $(TOP)/ext/recover/sqlite3recover.h tsrc/ + cp $(TOP)/ext/recover/dbdata.c tsrc/ + $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) + sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . diff --git a/manifest b/manifest index 4cd29a45bd..b6d1cb0513 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Small\sperformance\simprovement\sin\ssqlite3BtreeTransferRow(). -D 2022-11-21T13:35:00.374 +C Add\sMakefile.in\stargets\sfor\ssqlite3r.c\sand\ssqlite3r.h,\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension. +D 2022-11-22T16:12:53.393 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 78e4c4916f2c3993a8a454018745ff02094a8029d449d0c22db98f1cf8260420 +F Makefile.in 9a1228fd6406cc607958ccf1386a9a8f74f8255a7e98bc9c0a2792b56bd3a309 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc e7a564ceec71f0d9666031d5638cf0d4f88b050b44e8df5d32125137cd259ac0 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 947c5714ae9b15a41bc69421ed33f642ed438d09536d0eeecd8964cc05d740ce F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -1995,8 +1995,8 @@ F tool/mkshellc.tcl 02d0de8349ef830c0fb20d29680320bde2466e2ec422e5bd94c4317a7a7e F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f -F tool/mksqlite3c.tcl 4fc26a9bfa0c4505b203d7ca0cf086e75ebcd4d63eb719c82f37e3fecdf23d37 -F tool/mksqlite3h.tcl 1f5e4a1dbbbc43c83cc6e74fe32c6c620502240b66c7c0f33a51378e78fc4edf +F tool/mksqlite3c.tcl eb47021591b1ad4a6862e2cb5625f1ac67ec1e0c6db5ba3953c069c635284cf5 +F tool/mksqlite3h.tcl d391cff7cad0a372ee1406faee9ccc7dad9cb80a0c95cae0f73d10dd26e06762 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 @@ -2059,8 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07182dca9f2a4ffea1af0684c93e55e105465b2ee9820c70764e3e7bc1c28efc -R 06d7ab66cd3f5a521e55d8df4889b401 -U drh -Z a6fa3008dc4759fddba2bf18530ff99d +P dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 +R 1ce5997170cf4e38f2813ab1d1191f64 +T *branch * make-sqlite3r.c +T *sym-make-sqlite3r.c * +T -sym-trunk * +U dan +Z c67d4c65fcd4c164d590b57cc9ec79e4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e7e8aac79f..5f09b562a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 \ No newline at end of file +59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..29e61c7b16 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1066,9 +1066,11 @@ INCLUDE ../ext/expert/sqlite3expert.c #define SQLITE_SHELL_HAVE_RECOVER 0 #endif #if SQLITE_SHELL_HAVE_RECOVER -INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h +# ifndef SQLITE_HAVE_SQLITE3R +INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.c +# endif #endif #if defined(SQLITE_ENABLE_SESSION) diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 0cdf514e44..bc1aadd194 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -40,11 +40,14 @@ set help {Usage: tclsh mksqlite3c.tcl set addstatic 1 set linemacros 0 set useapicall 0 +set enable_recover 0 set srcdir tsrc for {set i 0} {$i<[llength $argv]} {incr i} { set x [lindex $argv $i] - if {[regexp {^-?-nostatic$} $x]} { + if {[regexp {^-?-enable-recover$} $x]} { + set enable_recover 1 + } elseif {[regexp {^-?-nostatic$} $x]} { set addstatic 0 } elseif {[regexp {^-?-linemacros(?:=([01]))?$} $x ma ulm]} { if {$ulm == ""} {set ulm 1} @@ -78,7 +81,9 @@ close $in # Open the output file and write a header comment at the beginning # of the file. # -set out [open sqlite3.c w] +set fname sqlite3.c +if {$enable_recover} { set fname sqlite3r.c } +set out [open $fname w] # Force the output to use unix line endings, even on Windows. fconfigure $out -translation lf set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] @@ -162,6 +167,7 @@ foreach hdr { vxworks.h wal.h whereInt.h + sqlite3recover.h } { set available_hdr($hdr) 1 } @@ -325,7 +331,7 @@ proc copy_file {filename} { # used subroutines first in order to help the compiler find # inlining opportunities. # -foreach file { +set flist { sqliteInt.h os_common.h ctime.c @@ -441,7 +447,11 @@ foreach file { sqlite3session.c fts5.c stmt.c -} { +} +if {$enable_recover} { + lappend flist sqlite3recover.c dbdata.c +} +foreach file $flist { copy_file $srcdir/$file } diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 9078a15753..bd579c28b0 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -40,9 +40,16 @@ set TOP [lindex $argv 0] # set useapicall 0 +# Include sqlite3recover.h? +# +set enable_recover 0 + if {[lsearch -regexp [lrange $argv 1 end] {^-+useapicall}] != -1} { set useapicall 1 } +if {[lsearch -regexp [lrange $argv 1 end] {^-+enable-recover}] != -1} { + set enable_recover 1 +} # Get the SQLite version number (ex: 3.6.18) from the $TOP/VERSION file. # @@ -84,6 +91,9 @@ set filelist [subst { $TOP/ext/session/sqlite3session.h $TOP/ext/fts5/fts5.h }] +if {$enable_recover} { + lappend filelist "$TOP/ext/recover/sqlite3recover.h" +} # These are the functions that accept a variable number of arguments. They # always need to use the "cdecl" calling convention even when another calling From c9b7d943ad594bcf563b44405d8cea6256d37210 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 18:24:07 +0000 Subject: [PATCH 14/72] Fix the error message in the CLI for "no such VFS". [forum:/info/1217fdd6b3|Forum post 1217fdd6b3]. FossilOrigin-Name: 861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e1376b3753..4b77a98c23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slast\sfunction\spointer\sin\ssqlite3Apis\sinit.\sReported\sat\shttps://sqlite.org/forum/info/eba0faa96d -D 2022-11-22T15:41:32.493 +C Fix\sthe\serror\smessage\sin\sthe\sCLI\sfor\s"no\ssuch\sVFS".\n[forum:/info/1217fdd6b3|Forum\spost\s1217fdd6b3]. +D 2022-11-22T18:24:07.441 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 -R b2f32015706aa1808841626a0ff1391c -U larrybr -Z d9d2dcaeb03b2f8d48cf12bc3dcb589d +P 6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a +R c32ff36f085986b819f258da8a8464b1 +U drh +Z f12a671d42ea42f2f1054a78088f03c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1264b5622a..26acee9357 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a \ No newline at end of file +861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..7f7ec779d4 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11767,7 +11767,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( pVfs ){ sqlite3_vfs_register(pVfs, 1); }else{ - utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]); + utf8_printf(stderr, "no such VFS: \"%s\"\n", zVfs); exit(1); } } From 5d7aef16bffb21331e7724cf475c18ac8d2d6d0f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 19:49:16 +0000 Subject: [PATCH 15/72] Rename the SELECTTRACE macro to TREETRACE, so that is corresponds to the new CLI command. Renumber all of the bits in the bitmask used to enable various kinds of tracing, and add a trace bitmap decoder in sqliteInt.h. Changes to debugging logic only. No (intentional) changes to production code. FossilOrigin-Name: 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd --- manifest | 16 ++++---- manifest.uuid | 2 +- src/select.c | 104 ++++++++++++++++++++++++------------------------ src/sqliteInt.h | 4 +- src/window.c | 2 +- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 4b77a98c23..19b1249875 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\serror\smessage\sin\sthe\sCLI\sfor\s"no\ssuch\sVFS".\n[forum:/info/1217fdd6b3|Forum\spost\s1217fdd6b3]. -D 2022-11-22T18:24:07.441 +C Rename\sthe\sSELECTTRACE\smacro\sto\sTREETRACE,\sso\sthat\sis\scorresponds\sto\sthe\snew\nCLI\scommand.\s\sRenumber\sall\sof\sthe\sbits\sin\sthe\sbitmask\sused\sto\senable\nvarious\skinds\sof\stracing,\sand\sadd\sa\strace\sbitmap\sdecoder\sin\ssqliteInt.h.\nChanges\sto\sdebugging\slogic\sonly.\s\sNo\s(intentional)\schanges\sto\sproduction\scode. +D 2022-11-22T19:49:16.908 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d +F src/select.c 4c48373abb4e67129c36bc15d1f5a99a0dfd9534afeb539a2169a09ae91ccec9 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 42bcbb6cebef64c99068e7bc141a2541d40d65c7c26a9ca5ada4d83ca353555d +F src/sqliteInt.h 3bdf90cb7578dfccfd3e8c0f9c8b0eecc5280f5813891d6b3016cd6f1bdb1f7e F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -732,7 +732,7 @@ F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae -F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 +F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test b5c19d504dec222c0dc66642673d23dce915d35737b68e74d9f237b80493eb53 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a -R c32ff36f085986b819f258da8a8464b1 +P 861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b +R 513d506718a1d19eb8b1625991fbf39f U drh -Z f12a671d42ea42f2f1054a78088f03c4 +Z 151ecb78555215509ebdea4705787b10 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 26acee9357..89e90a49a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b \ No newline at end of file +8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd \ No newline at end of file diff --git a/src/select.c b/src/select.c index f996ac0852..4d5bde9a31 100644 --- a/src/select.c +++ b/src/select.c @@ -2091,7 +2091,7 @@ void sqlite3GenerateColumnNames( if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; - SELECTTRACE(1,pParse,pSelect,("generating column names\n")); + TREETRACE(0x80,pParse,pSelect,("generating column names\n")); pTabList = pSelect->pSrc; pEList = pSelect->pEList; assert( v!=0 ); @@ -2877,7 +2877,7 @@ static int multiSelect( pPrior->iLimit = p->iLimit; pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); rc = sqlite3Select(pParse, pPrior, &dest); pPrior->pLimit = 0; if( rc ){ @@ -2895,7 +2895,7 @@ static int multiSelect( } } ExplainQueryPlan((pParse, 1, "UNION ALL")); - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -2948,7 +2948,7 @@ static int multiSelect( */ assert( !pPrior->pOrderBy ); sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); rc = sqlite3Select(pParse, pPrior, &uniondest); if( rc ){ goto multi_select_end; @@ -2968,7 +2968,7 @@ static int multiSelect( uniondest.eDest = op; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); rc = sqlite3Select(pParse, p, &uniondest); testcase( rc!=SQLITE_OK ); assert( p->pOrderBy==0 ); @@ -3029,7 +3029,7 @@ static int multiSelect( /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); rc = sqlite3Select(pParse, pPrior, &intersectdest); if( rc ){ goto multi_select_end; @@ -3046,7 +3046,7 @@ static int multiSelect( intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -4395,7 +4395,7 @@ static int flattenSubquery( } /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + TREETRACE(0x4,pParse,p,("flatten %u.%p from term %d\n", pSub->selId, pSub, iFrom)); /* Authorize the subquery */ @@ -4474,7 +4474,7 @@ static int flattenSubquery( if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; - SELECTTRACE(2,pParse,p,("compound-subquery flattener" + TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } assert( pSubitem->pSelect==0 ); @@ -4654,8 +4654,8 @@ static int flattenSubquery( sqlite3SelectDelete(db, pSub1); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + if( sqlite3TreeTrace & 0x4 ){ + TREETRACE(0x4,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6068,8 +6068,8 @@ static int selectExpander(Walker *pWalker, Select *p){ } } #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After result-set wildcard expansion:\n")); + if( sqlite3TreeTrace & 0x8 ){ + TREETRACE(0x8,pParse,p,("After result-set wildcard expansion:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6465,7 +6465,7 @@ static void havingToWhere(Parse *pParse, Select *p){ sqlite3WalkExpr(&sWalker, p->pHaving); #if TREETRACE_ENABLED if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){ - SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); + TREETRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6597,8 +6597,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ p->selFlags &= ~SF_Aggregate; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n")); + if( sqlite3TreeTrace & 0x200 ){ + TREETRACE(0x200,pParse,p,("After count-of-view optimization:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6698,8 +6698,8 @@ int sqlite3Select( assert( db->mallocFailed==0 ); if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); - if( sqlite3TreeTrace & 0x10100 ){ + TREETRACE(0x1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3TreeTrace & 0x10000 ){ if( (sqlite3TreeTrace & 0x10001)==0x10000 ){ sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d", __FILE__, __LINE__); @@ -6719,8 +6719,8 @@ int sqlite3Select( /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n")); - if( sqlite3TreeTrace & 0x100 ){ + TREETRACE(0x800,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3TreeTrace & 0x800 ){ sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); } #endif @@ -6740,8 +6740,8 @@ int sqlite3Select( assert( db->mallocFailed==0 ); assert( p->pEList!=0 ); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x104 ){ - SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); + if( sqlite3TreeTrace & 0x10 ){ + TREETRACE(0x10,pParse,p, ("after name resolution:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6782,8 +6782,8 @@ int sqlite3Select( goto select_end; } #if TREETRACE_ENABLED - if( p->pWin && (sqlite3TreeTrace & 0x108)!=0 ){ - SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + if( p->pWin && (sqlite3TreeTrace & 0x40)!=0 ){ + TREETRACE(0x40,pParse,p, ("after window rewrite:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6814,7 +6814,7 @@ int sqlite3Select( && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) && OptimizationEnabled(db, SQLITE_SimplifyJoin) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x1000,pParse,p, ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); assert( pItem->iCursor>=0 ); @@ -6870,7 +6870,7 @@ int sqlite3Select( && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x800,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); sqlite3ParserAddCleanup(pParse, (void(*)(sqlite3*,void*))sqlite3ExprListDelete, @@ -6925,8 +6925,8 @@ int sqlite3Select( if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end compound-select processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x400,pParse,p,("end compound-select processing\n")); + if( (sqlite3TreeTrace & 0x400)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6946,13 +6946,13 @@ int sqlite3Select( && propagateConstants(pParse, p) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + if( sqlite3TreeTrace & 0x2000 ){ + TREETRACE(0x2000,pParse,p,("After constant propagation:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif }else{ - SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } #ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION @@ -7025,15 +7025,15 @@ int sqlite3Select( && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p, + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); } zSavedAuthContext = pParse->zAuthContext; @@ -7148,8 +7148,8 @@ int sqlite3Select( sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + if( sqlite3TreeTrace & 0x8000 ){ + TREETRACE(0x8000,pParse,p,("After all FROM-clause analysis:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -7185,8 +7185,8 @@ int sqlite3Select( sDistinct.isTnct = 2; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + if( sqlite3TreeTrace & 0x20000 ){ + TREETRACE(0x20000,pParse,p,("Transform DISTINCT into GROUP BY:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -7272,7 +7272,7 @@ int sqlite3Select( /* Begin the database scan. */ - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, p->pEList, p, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; @@ -7289,7 +7289,7 @@ int sqlite3Select( sSort.pOrderBy = 0; } } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -7328,7 +7328,7 @@ int sqlite3Select( /* End the database scan loop. */ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); } }else{ @@ -7451,8 +7451,8 @@ int sqlite3Select( pAggInfo->mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); if( minMaxFlag ){ sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); @@ -7527,7 +7527,7 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 @@ -7537,7 +7537,7 @@ int sqlite3Select( goto select_end; } eDist = sqlite3WhereIsDistinct(pWInfo); - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be @@ -7586,7 +7586,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); @@ -7664,7 +7664,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); } @@ -7826,13 +7826,13 @@ int sqlite3Select( assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); eDist = sqlite3WhereIsDistinct(pWInfo); updateAccumulator(pParse, regAcc, pAggInfo, eDist); if( eDist!=WHERE_DISTINCT_NOOP ){ @@ -7846,7 +7846,7 @@ int sqlite3Select( if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); } - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, pAggInfo); } @@ -7907,8 +7907,8 @@ select_end: #endif #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3TreeTrace & 0x40000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6ff49ab5d0..9585bd4b2e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1029,12 +1029,12 @@ extern u32 sqlite3TreeTrace; && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE) \ || defined(SQLITE_ENABLE_TREETRACE)) # define TREETRACE_ENABLED 1 -# define SELECTTRACE(K,P,S,X) \ +# define TREETRACE(K,P,S,X) \ if(sqlite3TreeTrace&(K)) \ sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ sqlite3DebugPrintf X #else -# define SELECTTRACE(K,P,S,X) +# define TREETRACE(K,P,S,X) # define TREETRACE_ENABLED 0 #endif diff --git a/src/window.c b/src/window.c index f13ea8b027..1ed3e49214 100644 --- a/src/window.c +++ b/src/window.c @@ -1069,7 +1069,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ pSub = sqlite3SelectNew( pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 ); - SELECTTRACE(1,pParse,pSub, + TREETRACE(0x40,pParse,pSub, ("New window-function subquery in FROM clause of (%u/%p)\n", p->selId, p)); p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); From c7c5b8a508a2f941b00d43f5f1aa413e7cc1b171 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 19:56:54 +0000 Subject: [PATCH 16/72] Include the treetrace bitmap comment accidentally omitted from the previous check-in. FossilOrigin-Name: db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 19b1249875..d995c765ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sthe\sSELECTTRACE\smacro\sto\sTREETRACE,\sso\sthat\sis\scorresponds\sto\sthe\snew\nCLI\scommand.\s\sRenumber\sall\sof\sthe\sbits\sin\sthe\sbitmask\sused\sto\senable\nvarious\skinds\sof\stracing,\sand\sadd\sa\strace\sbitmap\sdecoder\sin\ssqliteInt.h.\nChanges\sto\sdebugging\slogic\sonly.\s\sNo\s(intentional)\schanges\sto\sproduction\scode. -D 2022-11-22T19:49:16.908 +C Include\sthe\streetrace\sbitmap\scomment\saccidentally\somitted\sfrom\sthe\sprevious\ncheck-in. +D 2022-11-22T19:56:54.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb2 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 3bdf90cb7578dfccfd3e8c0f9c8b0eecc5280f5813891d6b3016cd6f1bdb1f7e +F src/sqliteInt.h 6a24230f2928b3d1d9b0fdbedb98c862b828a4b1a9170306108e74cd6277f476 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b -R 513d506718a1d19eb8b1625991fbf39f +P 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd +R cda0c33c9442df99ec4e280c19853a61 U drh -Z 151ecb78555215509ebdea4705787b10 +Z 4e00dfd5e0c8561123895f1dda8a08f5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 89e90a49a0..e6bb7d9672 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd \ No newline at end of file +db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9585bd4b2e..86f33dec9e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1038,6 +1038,29 @@ extern u32 sqlite3TreeTrace; # define TREETRACE_ENABLED 0 #endif +/* TREETRACE flag meanings: +** +** 0x00000001 Beginning and end of SELECT processing +** 0x00000002 WHERE clause processing +** 0x00000004 Query flattener +** 0x00000008 Result-set wildcard expansion +** 0x00000010 Query name resolution +** 0x00000020 Aggregate analysis +** 0x00000040 Window functions +** 0x00000080 Generated column names +** 0x00000100 Move HAVING terms into WHERE +** 0x00000200 Count-of-view optimization +** 0x00000400 Compound SELECT processing +** 0x00000800 Drop superfluous ORDER BY +** 0x00001000 LEFT JOIN simplifies to JOIN +** 0x00002000 Constant propagation +** 0x00004000 Push-down optimization +** 0x00008000 After all FROM-clause analysis +** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing +** 0x00020000 Transform DISTINCT into GROUP BY +** 0x00040000 SELECT tree dump after all code has been generated +*/ + /* ** Macros for "wheretrace" */ From c62052217504fe651505328c32ea024c554e8a08 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 22 Nov 2022 20:04:00 +0000 Subject: [PATCH 17/72] Avoid naming collision between the sha1 and shathree extensions. FossilOrigin-Name: 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 --- ext/misc/shathree.c | 8 ++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index f80dd855da..765c691811 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -531,7 +531,7 @@ static void sha3Func( /* Compute a string using sqlite3_vsnprintf() with a maximum length ** of 50 bytes and add it to the hash. */ -static void hash_step_vformat( +static void sha3_step_vformat( SHA3Context *p, /* Add content to this context */ const char *zFormat, ... @@ -627,7 +627,7 @@ static void sha3QueryFunc( z = sqlite3_sql(pStmt); if( z ){ n = (int)strlen(z); - hash_step_vformat(&cx,"S%d:",n); + sha3_step_vformat(&cx,"S%d:",n); SHA3Update(&cx,(unsigned char*)z,n); } @@ -671,14 +671,14 @@ static void sha3QueryFunc( case SQLITE_TEXT: { int n2 = sqlite3_column_bytes(pStmt, i); const unsigned char *z2 = sqlite3_column_text(pStmt, i); - hash_step_vformat(&cx,"T%d:",n2); + sha3_step_vformat(&cx,"T%d:",n2); SHA3Update(&cx, z2, n2); break; } case SQLITE_BLOB: { int n2 = sqlite3_column_bytes(pStmt, i); const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - hash_step_vformat(&cx,"B%d:",n2); + sha3_step_vformat(&cx,"B%d:",n2); SHA3Update(&cx, z2, n2); break; } diff --git a/manifest b/manifest index d995c765ef..ee115293de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Include\sthe\streetrace\sbitmap\scomment\saccidentally\somitted\sfrom\sthe\sprevious\ncheck-in. -D 2022-11-22T19:56:54.514 +C Avoid\snaming\scollision\sbetween\sthe\ssha1\sand\sshathree\sextensions. +D 2022-11-22T20:04:00.704 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -322,7 +322,7 @@ F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d385 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c 8d79354f2c3d46b95ee21272a07cf0bcabb58d1f2b06d9e7b8a31dca1dacb3e5 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 7b17615869a495659f1569ada1d8d3d21b4a24614f2746d93cc87ef7c0b6b36d +F ext/misc/shathree.c 9b7af7b7d55b27c5bbd16548b67fe6aa47a819e71bc6a80a456144f86f4e834d F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f F ext/misc/sqlar.c 0ace5d3c10fe736dc584bf1159a36b8e2e60fab309d310cd8a0eecd9036621b6 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd -R cda0c33c9442df99ec4e280c19853a61 -U drh -Z 4e00dfd5e0c8561123895f1dda8a08f5 +P db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 +R 028f4214066f285c3501b4fa202e9f4b +U mistachkin +Z 9f8febbd78f0b476308550c8b424fd90 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e6bb7d9672..7b2a4dc5e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 \ No newline at end of file +9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 \ No newline at end of file From b23f1572ab9c11e524023678a5347152aae48b04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 20:37:41 +0000 Subject: [PATCH 18/72] Add the stub function: optimizeAggregateUsingIndexedExpr(). The hope is that we can fill this in with a routine that does useful optimizations. FossilOrigin-Name: d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8586f2587b..e809fe26ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sand\senhancements\sinto\sthe\sagg-with-indexed-expr\sbranch. -D 2022-11-22T19:51:16.237 +C Add\sthe\sstub\sfunction:\soptimizeAggregateUsingIndexedExpr().\s\sThe\shope\sis\sthat\nwe\scan\sfill\sthis\sin\swith\sa\sroutine\sthat\sdoes\suseful\soptimizations. +D 2022-11-22T20:37:41.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 35c2b1b4d82190f202887620f99070c0d414c3e030e466032797792cffdbd9f9 +F src/select.c dfc504356e15bb8585d8535a927a72512d1d7a01f9074824964f1781e286231b F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd -R e6ec32714f5519d3f42f7a87b92d7eba +P 070634781a5eb41f96b001e48b064e3cd8c82314f576335eb1fcd43792179291 +R 5bcfbb263a340b6c57659d80015cbb33 U drh -Z 31a3ccb601c45d898a857131ff2f396b +Z 28092ed1d737fad311ecc59737148e5d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1556ed2eb3..0a7b42af87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -070634781a5eb41f96b001e48b064e3cd8c82314f576335eb1fcd43792179291 \ No newline at end of file +d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 946649ad0b..ecd1791060 100644 --- a/src/select.c +++ b/src/select.c @@ -6654,6 +6654,34 @@ static void printAggInfo(AggInfo *pAggInfo){ } #endif /* TREETRACE_ENABLED */ +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static int optimizeAggregateUsingIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT being coded */ + AggInfo *pAggInfo /* The aggregate info */ +){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x100000 ){ + IndexedExpr *pIEpr; + TREETRACE(0x1000000, pParse, pSelect, + ("Attempting to optimize AggInfo for Indexed Exprs\n")); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("cur=%d\n", pIEpr->iDataCur); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#endif + return 0; +} + + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7535,6 +7563,9 @@ int sqlite3Select( sqlite3ExprListDelete(db, pDistinct); goto select_end; } + if( pParse->pIdxEpr ){ + optimizeAggregateUsingIndexedExpr(pParse, p, pAggInfo); + } assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); From c6138e970e4afca027635e97d0516a58e5c026a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 14:13:39 +0000 Subject: [PATCH 19/72] Further foundation prep work prior to starting to flesh-out the optimizeAggregateUseOfIndexedExpr() routine. FossilOrigin-Name: 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e --- manifest | 12 ++-- manifest.uuid | 2 +- src/select.c | 171 +++++++++++++++++++++++++++++--------------------- 3 files changed, 108 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 768c50fa57..1ffe8b4bb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sand\sfixes\sfrom\strunk. -D 2022-11-22T20:58:18.661 +C Further\sfoundation\sprep\swork\sprior\sto\sstarting\sto\sflesh-out\sthe\noptimizeAggregateUseOfIndexedExpr()\sroutine. +D 2022-11-23T14:13:39.215 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6f5403b8f1849d2a8ba152dd790ec189bba18242157904ddca77707e209265d4 +F src/select.c 1d3f8a5b510f8e2859c7f800babb5a63802789f93c9c29498312b37655dac1b8 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R 55f08c65335ec615b6caa516a7697c9d +P f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 +R e49e6155bccbafba580b14baaea3db86 U drh -Z dbc39758acc6c225892e0cac1e391b2a +Z b6500d1e8d8b2720badce4287d0e1594 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9c7d27e3fe..f88bcea3d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 \ No newline at end of file +23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e \ No newline at end of file diff --git a/src/select.c b/src/select.c index f71fa3197e..8f2517ad4f 100644 --- a/src/select.c +++ b/src/select.c @@ -6204,9 +6204,106 @@ void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +#if TREETRACE_ENABLED /* -** Assign register numbers to all pAggInfo->aCol[] and pAggInfo->aFunc[] -** entries. +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + ii, AggInfoFuncReg(pAggInfo,ii)); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + Parse *pParse, + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + IndexedExpr *pIEpr; + TREETRACE(0x80000, pParse, pSelect, + ("Attempting to optimize AggInfo for Indexed Exprs\n")); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + (void)pSelect; /* Suppress unused-parameter warnings */ +#endif + pAggInfo->nColumn = pAggInfo->nAccumulator; + analyzeAggFuncArgs(pParse, pAggInfo, pNC); +} + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ assert( pAggInfo!=0 ); @@ -6630,60 +6727,6 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ return 0; } -#if TREETRACE_ENABLED -/* -** Display all information about an AggInfo object -*/ -static void printAggInfo(AggInfo *pAggInfo){ - int ii; - for(ii=0; iinColumn; ii++){ - struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; - sqlite3DebugPrintf( - "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" - " iSorterColumn=%d %s\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), - pCol->iSorterColumn, - ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=\n", - ii, AggInfoFuncReg(pAggInfo,ii)); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } -} -#endif /* TREETRACE_ENABLED */ - -/* -** An index on expressions is being used in the inner loop of an -** aggregate query with a GROUP BY clause. This routine attempts -** to adjust the AggInfo object to take advantage of index and to -** perhaps use the index as a covering index. -** -*/ -static int optimizeAggregateUsingIndexedExpr( - Parse *pParse, /* Parsing context */ - Select *pSelect, /* The SELECT being coded */ - AggInfo *pAggInfo /* The aggregate info */ -){ -#if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x80000 ){ - IndexedExpr *pIEpr; - TREETRACE(0x80000, pParse, pSelect, - ("Attempting to optimize AggInfo for Indexed Exprs\n")); - for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ - printf("data-cursor=%d index={%d,%d}\n", - pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); - sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); - } - printAggInfo(pAggInfo); - } -#endif - return 0; -} - - /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7465,19 +7508,7 @@ int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; inFunc; i++){ - Expr *pExpr = pAggInfo->aFunc[i].pFExpr; - assert( ExprUseXList(pExpr) ); - sNC.ncFlags |= NC_InAggFunc; - sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); -#ifndef SQLITE_OMIT_WINDOWFUNC - assert( !IsWindowFunc(pExpr) ); - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); - } -#endif - sNC.ncFlags &= ~NC_InAggFunc; - } + analyzeAggFuncArgs(pParse, pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ @@ -7566,7 +7597,7 @@ int sqlite3Select( goto select_end; } if( pParse->pIdxEpr ){ - optimizeAggregateUsingIndexedExpr(pParse, p, pAggInfo); + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); } assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); From 3eaaec945c4cc3b35e9bd3c0f9a133c7f94cae1f Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 15:33:49 +0000 Subject: [PATCH 20/72] OPFS concurrency tester: ensure that the work interval timer does not overlap with the work time. FossilOrigin-Name: 8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 --- ext/wasm/tests/opfs/concurrency/worker.js | 5 +++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index c315508e0b..19b0a068e7 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -70,11 +70,12 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; if(1){/*use setInterval()*/ - interval.handle = setInterval(async ()=>{ + setTimeout(async function timer(){ await doWork(); if(interval.error || maxIterations === interval.count){ - clearInterval(interval.handle); finish(); + }else{ + setTimeout(timer, interval.delay); } }, interval.delay); }else{ diff --git a/manifest b/manifest index ee115293de..0133034207 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\snaming\scollision\sbetween\sthe\ssha1\sand\sshathree\sextensions. -D 2022-11-22T20:04:00.704 +C OPFS\sconcurrency\stester:\sensure\sthat\sthe\swork\sinterval\stimer\sdoes\snot\soverlap\swith\sthe\swork\stime. +D 2022-11-23T15:33:49.725 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js df065bb386ff994951f7fbdd76e12f16e58fbef0e929b2caf74553359da40afc +F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 -R 028f4214066f285c3501b4fa202e9f4b -U mistachkin -Z 9f8febbd78f0b476308550c8b424fd90 +P 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 +R f934382a93ebec7212ddc22bd89cd134 +U stephan +Z 5fd98b4beaa4b9278c801856cb1d0dd1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7b2a4dc5e4..04ede9801e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 \ No newline at end of file +8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 \ No newline at end of file From 91a81316b9b184126b3546b332b36749209d34bf Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 15:52:52 +0000 Subject: [PATCH 21/72] Remove a bit of over-cleverness which breaks loading of sqlite3.js in some main-thread cases. FossilOrigin-Name: 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 --- ext/wasm/api/extern-post-js.js | 3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index b327837814..cace6ed51c 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -59,9 +59,6 @@ const toExportForES6 = li.pop(); initModuleState.sqlite3Dir = li.join('/') + '/'; } - if(initModuleState.sqlite3Dir){ - initModuleState.sqlite3Dir = initModuleState.sqlite3Dir.replace(/[/]{2,}/g,'/'); - } self.sqlite3InitModule = (...args)=>{ //console.warn("Using replaced sqlite3InitModule()",self.location); diff --git a/manifest b/manifest index 0133034207..eab1abce2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sconcurrency\stester:\sensure\sthat\sthe\swork\sinterval\stimer\sdoes\snot\soverlap\swith\sthe\swork\stime. -D 2022-11-23T15:33:49.725 +C Remove\sa\sbit\sof\sover-cleverness\swhich\sbreaks\sloading\sof\ssqlite3.js\sin\ssome\smain-thread\scases. +D 2022-11-23T15:52:52.817 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 31400dd1c0ae3458a0e6510229e59318e45eac402a75dd703c2950b9b5758b46 +F ext/wasm/api/extern-post-js.js 59e52f579cd3a332d73dae94c91b9579daafb10dd6ada03803f1afa6bdad7689 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R f934382a93ebec7212ddc22bd89cd134 +P 8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 +R 48a8d083139c4365079fe8210cacb510 U stephan -Z 5fd98b4beaa4b9278c801856cb1d0dd1 +Z c7336980b293a293ed2df30baba9b924 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 04ede9801e..9666b96008 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 \ No newline at end of file +220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 \ No newline at end of file From ad1285c5c0be7eb92cc44a3357be71507c3c07f2 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 16:39:07 +0000 Subject: [PATCH 22/72] Initial infrastructure for adding a mode to the OPFS VFS which causes implicit locks to be released ASAP, which increases concurrency at the cost of performance. FossilOrigin-Name: c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 --- ext/wasm/api/sqlite3-api-oo1.js | 5 +- ext/wasm/api/sqlite3-api-opfs.js | 1 + ext/wasm/api/sqlite3-opfs-async-proxy.js | 123 ++++++++++++++++------ ext/wasm/tests/opfs/concurrency/worker.js | 6 +- manifest | 21 ++-- manifest.uuid | 2 +- 6 files changed, 112 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 02ce9c0ced..bc32ec1067 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -200,9 +200,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){ const arg = {}; - if(1===arguments.length && 'object'===typeof arguments[0]){ - const x = arguments[0]; - Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){ + Object.assign(arg, arguments[0]); if(undefined===arg.flags) arg.flags = 'c'; if(undefined===arg.vfs) arg.vfs = null; if(undefined===arg.filename) arg.filename = ':memory:'; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb4c923ab..53f68a1d58 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -348,6 +348,7 @@ const installOpfsVfs = function callee(options){ 'SQLITE_NOTFOUND', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', 'SQLITE_OPEN_READONLY' ].forEach((k)=>{ if(undefined === (state.sq3Codes[k] = capi[k])){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index c208932e17..cbe0549b3e 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -105,7 +105,7 @@ metrics.dump = ()=>{ */ const __openFiles = Object.create(null); /** - __autoLocks is a Set of sqlite3_file pointers (integers) which were + __implicitLocks is a Set of sqlite3_file pointers (integers) which were "auto-locked". i.e. those for which we obtained a sync access handle without an explicit xLock() call. Such locks will be released during db connection idle time, whereas a sync access @@ -117,7 +117,7 @@ const __openFiles = Object.create(null); penalty: speedtest1 benchmarks take up to 4x as long. By delaying the lock release until idle time, the hit is negligible. */ -const __autoLocks = new Set(); +const __implicitLocks = new Set(); /** Expects an OPFS file path. It gets resolved, such that ".." @@ -166,7 +166,7 @@ const closeSyncHandle = async (fh)=>{ const h = fh.syncHandle; delete fh.syncHandle; delete fh.xLock; - __autoLocks.delete(fh.fid); + __implicitLocks.delete(fh.fid); return h.close(); } }; @@ -190,10 +190,10 @@ const closeSyncHandleNoThrow = async (fh)=>{ }; /* Release all auto-locks. */ -const closeAutoLocks = async ()=>{ - if(__autoLocks.size){ +const releaseImplicitLocks = async ()=>{ + if(__implicitLocks.size){ /* Release all auto-locks. */ - for(const fid of __autoLocks){ + for(const fid of __implicitLocks){ const fh = __openFiles[fid]; await closeSyncHandleNoThrow(fh); log("Auto-unlocked",fid,fh.filenameAbs); @@ -201,6 +201,32 @@ const closeAutoLocks = async ()=>{ } }; +/** + If true, any routine which implicitly acquires a sync access handle + (i.e. an OPFS lock) will release that locks at the end of the call + which acquires it. If false, such "autolocks" are not released + until the VFS is idle for some brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). +*/ +state.defaultReleaseImplicitLocks = false; + +/** + An experiment in improving concurrency by freeing up implicit locks + sooner. This is known to impact performance dramatically but it has + also shown to improve concurrency considerably. + + If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, + this routine returns closeSyncHandleNoThrow(), else it is a no-op. +*/ +const releaseImplicitLock = async (fh)=>{ + if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ + return closeSyncHandleNoThrow(fh); + } +}; + /** An error class specifically for use with getSyncHandle(), the goal of which is to eventually be able to distinguish unambiguously @@ -246,7 +272,7 @@ GetSyncHandleError.convertRc = (e,rc)=>{ still fails at that point it will give up and propagate the exception. */ -const getSyncHandle = async (fh)=>{ +const getSyncHandle = async (fh,opName)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); @@ -262,20 +288,21 @@ const getSyncHandle = async (fh)=>{ }catch(e){ if(i === maxTries){ throw new GetSyncHandleError( - e, "Error getting sync handle.",maxTries, + e, "Error getting sync handle for",opName+"().",maxTries, "attempts failed.",fh.filenameAbs ); } - warn("Error getting sync handle. Waiting",ms, + warn("Error getting sync handle for",opName+"(). Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - await closeAutoLocks(); + //await releaseImplicitLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } - log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + log("Got",opName+"() sync handle for",fh.filenameAbs, + 'in',performance.now() - t,'ms'); if(!fh.xLock){ - __autoLocks.add(fh.fid); - log("Auto-locked",fh.fid,fh.filenameAbs); + __implicitLocks.add(fh.fid); + log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); } } return fh.syncHandle; @@ -409,7 +436,7 @@ const vfsAsyncImpls = { xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; mTimeStart(opName); - __autoLocks.delete(fid); + __implicitLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; wTimeStart(opName); @@ -474,13 +501,14 @@ const vfsAsyncImpls = { wTimeStart('xFileSize'); try{ affirmLocked('xFileSize',fh); - const sz = await (await getSyncHandle(fh)).getSize(); + const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); rc = 0; }catch(e){ state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xFileSize', rc); mTimeEnd(); @@ -495,8 +523,8 @@ const vfsAsyncImpls = { if( !fh.syncHandle ){ wTimeStart('xLock'); try { - await getSyncHandle(fh); - __autoLocks.delete(fid); + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); }catch(e){ state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); @@ -511,7 +539,6 @@ const vfsAsyncImpls = { flags/*SQLITE_OPEN_...*/){ const opName = 'xOpen'; mTimeStart(opName); - const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); wTimeStart('xOpen'); try{ @@ -526,14 +553,8 @@ const vfsAsyncImpls = { return; } const hFile = await hDir.getFileHandle(filenamePart, {create}); - /** - wa-sqlite, at this point, grabs a SyncAccessHandle and - assigns it to the syncHandle prop of the file state - object, but only for certain cases and it's unclear why it - places that limitation on it. - */ wTimeEnd(); - __openFiles[fid] = Object.assign(Object.create(null),{ + const fh = Object.assign(Object.create(null),{ fid: fid, filenameAbs: filename, filenamePart: filenamePart, @@ -542,8 +563,47 @@ const vfsAsyncImpls = { sabView: state.sabFileBufView, readOnly: create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), - deleteOnClose: deleteOnClose + deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); + fh.releaseImplicitLocks = + state.defaultReleaseImplicitLocks + /* TODO: check URI flags for "opfs-auto-unlock". First we need to + reshape the API a bit to be able to pass those on to here + from the other half of the proxy. */; + /*if(fh.releaseImplicitLocks){ + console.warn("releaseImplicitLocks is ON for",fh); + }*/ + if(0 /* this block is modelled after something wa-sqlite + does but it leads to horrible contention on journal files. */ + && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ + /* sqlite does not lock these files, so go ahead and grab an OPFS + lock. + + Regarding "immutable": that flag is not _really_ applicable + here. It's intended for use on read-only media. If, + however, a file is opened with that flag but changes later + (which can happen if we _don't_ grab a sync handle here) + then sqlite may misbehave. + + Regarding "nolock": ironically, the nolock flag forces us + to lock the file up front. "nolock" tells sqlite to _not_ + use its locking API, but OPFS requires a lock to perform + most of the operations performed in this file. If we don't + grab that lock up front, another handle could end up grabbing + it and mutating the database out from under our nolocked'd + handle. In the interest of preventing corruption, at the cost + of decreased concurrency, we have to lock it for the duration + of this file handle. + + https://www.sqlite.org/uri.html + */ + fh.xLock = "atOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; + await getSyncHandle(fh,'xOpen'); + } + __openFiles[fid] = fh; storeAndNotify(opName, 0); }catch(e){ wTimeEnd(); @@ -560,7 +620,7 @@ const vfsAsyncImpls = { try{ affirmLocked('xRead',fh); wTimeStart('xRead'); - nRead = (await getSyncHandle(fh)).read( + nRead = (await getSyncHandle(fh,'xRead')).read( fh.sabView.subarray(0, n), {at: Number(offset64)} ); @@ -575,6 +635,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } + await releaseImplicitLock(fh); storeAndNotify('xRead',rc); mTimeEnd(); }, @@ -603,12 +664,13 @@ const vfsAsyncImpls = { try{ affirmLocked('xTruncate',fh); affirmNotRO('xTruncate', fh); - await (await getSyncHandle(fh)).truncate(size); + await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xTruncate',rc); mTimeEnd(); @@ -640,7 +702,7 @@ const vfsAsyncImpls = { affirmLocked('xWrite',fh); affirmNotRO('xWrite', fh); rc = ( - n === (await getSyncHandle(fh)) + n === (await getSyncHandle(fh,'xWrite')) .write(fh.sabView.subarray(0, n), {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; @@ -649,6 +711,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xWrite',rc); mTimeEnd(); @@ -783,7 +846,7 @@ const waitLoop = async function f(){ if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ - await closeAutoLocks(); + await releaseImplicitLocks(); continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 19b0a068e7..9b4898f4c3 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -43,7 +43,11 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.opfs.OpfsDb(dbName,'c'); + db = new sqlite3.oo1.DB({ + filename: 'file:'+dbName, + flags: 'c', + vfs: 'opfs' + }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ db.exec([ diff --git a/manifest b/manifest index 3ecb281109..d6e43cfa43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMakefile.in\sto\sinclude\snew\starget\s"sqlite3r.c".\sFor\sgenerating\s"sqlite3r.c"\sand\s"sqlite3r.h",\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension.\sTo\sbuild\sthe\sshell\stool\sagainst\sthese\sfiles,\sadd\s-DSQLITE_HAVE_SQLITE3R. -D 2022-11-23T16:08:49.765 +C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. +D 2022-11-23T16:39:07.866 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,12 +501,12 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 -F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b +F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 +F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de +F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -R 6b5395a59674684ae7409c1a67752bd1 -T +closed 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -U dan -Z 08a51ae8157c87edcee2b45d299f4155 +P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 +R 5ffa2a11b06c4a44d92d27455a2fc3e8 +U stephan +Z 70f5adf76d87ddf8a488ff2bcd290afb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c19a5133e..54a2776a7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 \ No newline at end of file +c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file From 590f013a87609ea33abc718aab71c8785a7213f9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 17:56:00 +0000 Subject: [PATCH 23/72] This attempt at modifying AggInfo to make use of indexed expressions does not work. It gets an incorrect answer for the test case shown in the ticket. FossilOrigin-Name: 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 --- manifest | 16 ++--- manifest.uuid | 2 +- src/expr.c | 157 ++++++++++++++++++++++++++++++------------------ src/select.c | 21 +++++-- src/sqliteInt.h | 1 - 5 files changed, 122 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index 1ffe8b4bb7..294477b047 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sfoundation\sprep\swork\sprior\sto\sstarting\sto\sflesh-out\sthe\noptimizeAggregateUseOfIndexedExpr()\sroutine. -D 2022-11-23T14:13:39.215 +C This\sattempt\sat\smodifying\sAggInfo\sto\smake\suse\sof\sindexed\sexpressions\sdoes\snot\nwork.\s\sIt\sgets\san\sincorrect\sanswer\sfor\sthe\stest\scase\sshown\sin\sthe\sticket. +D 2022-11-23T17:56:00.136 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 63cce2c219748d8f8e00a30e4dc43d65b686b62489f154eebe892933bfb4a249 +F src/expr.c 527fc56468f75c380a34d38426cc1deece693a3cc6139fb45544923ff461d569 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 1d3f8a5b510f8e2859c7f800babb5a63802789f93c9c29498312b37655dac1b8 +F src/select.c 8ac7f73d40d5615cdb73660a85e05ee26943a618ec5c7262160994f263fa7178 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 894f7d98b1104fecb2ca2f457a4c598944250d9462eb433f12bd42b1080d525f +F src/sqliteInt.h f4917d663170308601d794cdff7b8cf9704c4bafe03d7e926a156be835e29f29 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 -R e49e6155bccbafba580b14baaea3db86 +P 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e +R f3df5c5148120a1bf76a131c870fdc09 U drh -Z b6500d1e8d8b2720badce4287d0e1594 +Z 97820e8b433aee285f0d7a5d438ada65 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f88bcea3d3..c7c9044f2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e \ No newline at end of file +84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 42099536ed..db81983e4a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4135,7 +4135,7 @@ expr_code_doover: pCol->iSorterColumn, target); if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( ALWAYS(pTab!=0) ){ + }else if( pTab!=0 ){ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -6241,6 +6241,71 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ return i; } +/* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an idex of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} + /* ** This is the xExprCallback for a tree walker. It is used to ** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates @@ -6255,6 +6320,37 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( pNC->ncFlags & NC_UAggInfo ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( !ExprUseYTab(pExpr) ) break; + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { @@ -6266,66 +6362,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( ALWAYS(pSrcList!=0) ){ SrcItem *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable - && pCol->iColumn==pExpr->iColumn - && pExpr->op!=TK_IF_NULL_ROW - ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - assert( ExprUseYTab(pExpr) ); - pCol->pTab = pExpr->y.pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iSorterColumn = -1; - pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN - && pE->iTable==pExpr->iTable - && pE->iColumn==pExpr->iColumn - ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - if( pExpr->op==TK_COLUMN ){ - pExpr->op = TK_AGG_COLUMN; - } - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ diff --git a/src/select.c b/src/select.c index 8f2517ad4f..49bef86bab 100644 --- a/src/select.c +++ b/src/select.c @@ -6281,11 +6281,22 @@ static void optimizeAggregateUseOfIndexedExpr( AggInfo *pAggInfo, /* The aggregate info */ NameContext *pNC /* Name context used to resolve agg-func args */ ){ + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( pAggInfo->nSortingColumn>0 ){ + if( pAggInfo->nColumn==0 ){ + pAggInfo->nSortingColumn = 0; + }else{ + pAggInfo->nSortingColumn = + pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; + } + } + analyzeAggFuncArgs(pParse, pAggInfo, pNC); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x80000 ){ + if( sqlite3TreeTrace & 0x20 ){ IndexedExpr *pIEpr; - TREETRACE(0x80000, pParse, pSelect, - ("Attempting to optimize AggInfo for Indexed Exprs\n")); + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ printf("data-cursor=%d index={%d,%d}\n", pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); @@ -6296,8 +6307,6 @@ static void optimizeAggregateUseOfIndexedExpr( #else (void)pSelect; /* Suppress unused-parameter warnings */ #endif - pAggInfo->nColumn = pAggInfo->nAccumulator; - analyzeAggFuncArgs(pParse, pAggInfo, pNC); } /* @@ -7959,7 +7968,7 @@ select_end: if( pAggInfo && !db->mallocFailed ){ for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; - assert( pExpr!=0 ); + if( pExpr==0 ) continue; assert( pExpr->pAggInfo==pAggInfo ); assert( pExpr->iAgg==i ); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 74c997c6dd..3ecdb35fee 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1059,7 +1059,6 @@ extern u32 sqlite3TreeTrace; ** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated -** 0x00080000 Optimize Aggregates using indexed expressions */ /* From 2e30d95fb63e73cbae3cf07b0ac57d8d7e0321ba Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 18:51:04 +0000 Subject: [PATCH 24/72] Aggregates with GROUP BY now make use of expressions on indexes. This code works and gets the correct answer for the test case in the ticket. Lots more testing and documentation is needed, however. FossilOrigin-Name: 8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 6 ++++-- src/select.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 294477b047..5a5586e90e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sattempt\sat\smodifying\sAggInfo\sto\smake\suse\sof\sindexed\sexpressions\sdoes\snot\nwork.\s\sIt\sgets\san\sincorrect\sanswer\sfor\sthe\stest\scase\sshown\sin\sthe\sticket. -D 2022-11-23T17:56:00.136 +C Aggregates\swith\sGROUP\sBY\snow\smake\suse\sof\sexpressions\son\sindexes.\s\sThis\scode\nworks\sand\sgets\sthe\scorrect\sanswer\sfor\sthe\stest\scase\sin\sthe\sticket.\s\sLots\smore\ntesting\sand\sdocumentation\sis\sneeded,\showever. +D 2022-11-23T18:51:04.363 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 527fc56468f75c380a34d38426cc1deece693a3cc6139fb45544923ff461d569 +F src/expr.c 141af8139174010ab37591df6234a647ecc38b41f72ac6e2c128ebcf167e0bc0 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 8ac7f73d40d5615cdb73660a85e05ee26943a618ec5c7262160994f263fa7178 +F src/select.c f253214cbd2458f744b95fc6bf8dee103cecf4b051f620fd5f62b43dbab92ec1 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e -R f3df5c5148120a1bf76a131c870fdc09 +P 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 +R 2344df85f68d9987b55d50d38bac47c1 U drh -Z 97820e8b433aee285f0d7a5d438ada65 +Z 4c924f435822db601e489fbe4dcd4a9e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c7c9044f2a..56c84edce5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 \ No newline at end of file +8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index db81983e4a..c34a28dc47 100644 --- a/src/expr.c +++ b/src/expr.c @@ -53,7 +53,7 @@ char sqlite3ExprAffinity(const Expr *pExpr){ } op = pExpr->op; if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ assert( ExprUseYTab(pExpr) ); assert( pExpr->y.pTab!=0 ); return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); @@ -173,7 +173,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ int j; assert( ExprUseYTab(p) ); assert( p->y.pTab!=0 ); diff --git a/src/select.c b/src/select.c index 49bef86bab..dcb878dde9 100644 --- a/src/select.c +++ b/src/select.c @@ -6222,7 +6222,7 @@ static void printAggInfo(AggInfo *pAggInfo){ sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", ii, AggInfoFuncReg(pAggInfo,ii)); sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); } @@ -6309,6 +6309,40 @@ static void optimizeAggregateUseOfIndexedExpr( #endif } +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; inFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + /* ** Allocate a block of registers so that there is one register for each ** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first @@ -7669,6 +7703,18 @@ int sqlite3Select( pAggInfo->useSortingIdx = 1; } + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif + } + /* If the index or temporary table used by the GROUP BY sort ** will naturally deliver rows in the order required by the ORDER BY ** clause, cancel the ephemeral table open coded earlier. From e79cb67c3575d8254b36eba3f35dba2d62124e07 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 19:03:22 +0000 Subject: [PATCH 25/72] Add an experimental OPFS VFS-specific URI flag, opfs-unlock-asap, which tells the VFS to release implicit locks ASAP. This permits higher concurrency but hurts performance considerably. This may or may not be obsoleted by other concurrency-related experimentation. FossilOrigin-Name: d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 --- ext/wasm/api/sqlite3-api-opfs.js | 27 ++++++++++++++++++++++- ext/wasm/api/sqlite3-opfs-async-proxy.js | 25 +++++---------------- ext/wasm/tests/opfs/concurrency/worker.js | 2 +- manifest | 16 +++++++------- manifest.uuid | 2 +- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 53f68a1d58..deb8bf04e8 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -355,6 +355,25 @@ const installOpfsVfs = function callee(options){ toss("Maintenance required: not found:",k); } }); + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). "opfs-unlock-asap=1" enables + this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + If true, any async routine which implicitly acquires a sync + access handle (i.e. an OPFS lock) will release that locks at + the end of the call which acquires it. If false, such + "autolocks" are not released until the VFS is idle for some + brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). + */ + defaultUnlockAsap: false + }); /** Runs the given operation (by name) in the async worker @@ -845,9 +864,15 @@ const installOpfsVfs = function callee(options){ //xSleep is optionally defined below xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ mTimeStart('xOpen'); + let opfsFlags = 0; if(0===zName){ zName = randomFilename(); }else if('number'===typeof zName){ + if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ + /* -----------------------^^^^^ MUST pass the untranslated + C-string here. */ + opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; + } zName = wasm.cstringToJs(zName); } const fh = Object.create(null); @@ -855,7 +880,7 @@ const installOpfsVfs = function callee(options){ fh.filename = zName; fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; - const rc = opRun('xOpen', pFile, zName, flags); + const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); if(!rc){ /* Recall that sqlite3_vfs::xClose() will be called, even on error, unless pFile->pMethods is NULL. */ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index cbe0549b3e..b14494a0c9 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -201,18 +201,6 @@ const releaseImplicitLocks = async ()=>{ } }; -/** - If true, any routine which implicitly acquires a sync access handle - (i.e. an OPFS lock) will release that locks at the end of the call - which acquires it. If false, such "autolocks" are not released - until the VFS is idle for some brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). -*/ -state.defaultReleaseImplicitLocks = false; - /** An experiment in improving concurrency by freeing up implicit locks sooner. This is known to impact performance dramatically but it has @@ -536,7 +524,8 @@ const vfsAsyncImpls = { mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, - flags/*SQLITE_OPEN_...*/){ + flags/*SQLITE_OPEN_...*/, + opfsFlags/*OPFS_...*/){ const opName = 'xOpen'; mTimeStart(opName); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); @@ -566,13 +555,8 @@ const vfsAsyncImpls = { deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); fh.releaseImplicitLocks = - state.defaultReleaseImplicitLocks - /* TODO: check URI flags for "opfs-auto-unlock". First we need to - reshape the API a bit to be able to pass those on to here - from the other half of the proxy. */; - /*if(fh.releaseImplicitLocks){ - console.warn("releaseImplicitLocks is ON for",fh); - }*/ + (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) + || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite does but it leads to horrible contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ @@ -887,6 +871,7 @@ navigator.storage.getDirectory().then(function(d){ state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); state.opIds = opt.opIds; state.sq3Codes = opt.sq3Codes; + state.opfsFlags = opt.opfsFlags; Object.keys(vfsAsyncImpls).forEach((k)=>{ if(!Number.isFinite(state.opIds[k])){ toss("Maintenance required: missing state.opIds[",k,"]"); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 9b4898f4c3..1a896e714e 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -44,7 +44,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName, + filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, flags: 'c', vfs: 'opfs' }); diff --git a/manifest b/manifest index d6e43cfa43..d9cc7cb9d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. -D 2022-11-23T16:39:07.866 +C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. +D 2022-11-23T19:03:22.450 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 +F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee +F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 +F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 -R 5ffa2a11b06c4a44d92d27455a2fc3e8 +P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 +R 37d44560e41f4eab685876e5b8c66c33 U stephan -Z 70f5adf76d87ddf8a488ff2bcd290afb +Z 1cab17b5c5d56b08d5be52571107da20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 54a2776a7a..8365ac9b1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file +d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file From 056a71562f192c7fcd4ca73a3133b60dcf5a83fd Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 20:49:08 +0000 Subject: [PATCH 26/72] OPFS concurrency test: add a URL flag to enable/disable unlock-asap mode. FossilOrigin-Name: 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 --- ext/wasm/tests/opfs/concurrency/test.js | 6 +++++- ext/wasm/tests/opfs/concurrency/worker.js | 11 +++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 27bc47b19d..cb9d4275be 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -63,6 +63,9 @@ options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 ) || 750; + options.unlockAsap = ( + urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 + ) || 0; const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); @@ -92,12 +95,13 @@ } }; - stdout("Launching",options.workerCount,"workers..."); + stdout("Launching",options.workerCount,"workers. Options:",options); workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + '&opfs-verbose='+options.opfsVerbose + + '&opfs-unlock-asap='+options.unlockAsap ); for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 1a896e714e..91aa0fa6fa 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -3,9 +3,12 @@ importScripts( ); self.sqlite3InitModule().then(async function(sqlite3){ const urlArgs = new URL(self.location.href).searchParams; - const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000); + const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ + }; const wPost = (type,...payload)=>{ - postMessage({type, worker: wName, payload}); + postMessage({type, worker: options.workerName, payload}); }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); @@ -44,7 +47,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c', vfs: 'opfs' }); @@ -66,7 +69,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ try{ db.exec({ sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", - bind: [wName, new Date().getTime()] + bind: [options.workerName, new Date().getTime()] }); //stdout("Set",prefix); }catch(e){ diff --git a/manifest b/manifest index d9cc7cb9d0..a6b0539bab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. -D 2022-11-23T19:03:22.450 +C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. +D 2022-11-23T20:49:08.427 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -553,8 +553,8 @@ F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b51073 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 +F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 +F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 -R 37d44560e41f4eab685876e5b8c66c33 +P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 +R ff8c3df82102d217683515f7779a95c4 U stephan -Z 1cab17b5c5d56b08d5be52571107da20 +Z 269769e48cdf286c8fb19455cdae3a52 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8365ac9b1a..bb0d09842b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file +1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file From 875db41afca4447f00ecbb554e5268017a4c8b87 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 21:03:22 +0000 Subject: [PATCH 27/72] Add optional zSchema argument to sqlite3_js_db_export(). FossilOrigin-Name: 9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa --- ext/wasm/api/sqlite3-api-prologue.js | 16 +++++++++++----- ext/wasm/api/sqlite3-wasm.c | 20 +++++++++++--------- manifest | 17 ++++++++--------- manifest.uuid | 2 +- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 8b2ce0936d..f9ad5b5127 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1310,14 +1310,17 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( Serializes the given `sqlite3*` pointer to a Uint8Array, as per sqlite3_serialize(). On success it returns a Uint8Array. On error it throws with a description of the problem. + + schema is the schema to serialize. It may be a WASM C-string + pointer or a JS string. If it is falsy, it defaults to "main". */ - capi.sqlite3_js_db_export = function(pDb){ + capi.sqlite3_js_db_export = function(pDb, schema=0){ if(!pDb) toss3('Invalid sqlite3* argument.'); if(!wasm.bigIntEnabled) toss3('BigInt64 support is not enabled.'); - const stack = wasm.pstack.pointer; + const scope = wasm.scopedAllocPush(); let pOut; try{ - const pSize = wasm.pstack.alloc(8/*i64*/ + wasm.ptrSizeof); + const pSize = wasm.scopedAlloc(8/*i64*/ + wasm.ptrSizeof); const ppOut = pSize + 8; /** Maintenance reminder, since this cost a full hour of grief @@ -1326,8 +1329,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( export reads a garbage size because it's not on an 8-byte memory boundary! */ + const zSchema = schema + ? (wasm.isPtr(schema) ? schema : wasm.scopedAllocCString(''+schema)) + : 0; let rc = wasm.exports.sqlite3_wasm_db_serialize( - pDb, ppOut, pSize, 0 + pDb, zSchema, ppOut, pSize, 0 ); if(rc){ toss3("Database serialization failed with code", @@ -1341,7 +1347,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return rc; }finally{ if(pOut) wasm.exports.sqlite3_free(pOut); - wasm.pstack.restore(stack); + wasm.scopedAllocPop(scope); } }; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index af5ed6bf76..67e97c4a03 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -916,25 +916,27 @@ int sqlite3_wasm_db_export_chunked( sqlite3* pDb, } /* -** A proxy for sqlite3_serialize() which serializes the "main" schema +** A proxy for sqlite3_serialize() which serializes the schema zSchema ** of pDb, placing the serialized output in pOut and nOut. nOut may be -** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If -** allocation of the serialized copy fails, SQLITE_NOMEM is returned. -** On success, 0 is returned and `*pOut` will contain a pointer to the -** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the -** database has no contiguous memory representation, in which case -** `*pOut` will be NULL but 0 will be returned. +** NULL. If zSchema is NULL then "main" is assumed. If pDb or pOut are +** NULL then SQLITE_MISUSE is returned. If allocation of the +** serialized copy fails, SQLITE_NOMEM is returned. On success, 0 is +** returned and `*pOut` will contain a pointer to the memory unless +** mFlags includes SQLITE_SERIALIZE_NOCOPY and the database has no +** contiguous memory representation, in which case `*pOut` will be +** NULL but 0 will be returned. ** ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ SQLITE_WASM_KEEP -int sqlite3_wasm_db_serialize( sqlite3 *pDb, unsigned char **pOut, +int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, + unsigned char **pOut, sqlite3_int64 *nOut, unsigned int mFlags ){ unsigned char * z; if( !pDb || !pOut ) return SQLITE_MISUSE; if(nOut) *nOut = 0; - z = sqlite3_serialize(pDb, "main", nOut, mFlags); + z = sqlite3_serialize(pDb, zSchema ? zSchema : "main", nOut, mFlags); if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){ *pOut = z; return 0; diff --git a/manifest b/manifest index 3ecb281109..12cb76e8ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMakefile.in\sto\sinclude\snew\starget\s"sqlite3r.c".\sFor\sgenerating\s"sqlite3r.c"\sand\s"sqlite3r.h",\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension.\sTo\sbuild\sthe\sshell\stool\sagainst\sthese\sfiles,\sadd\s-DSQLITE_HAVE_SQLITE3R. -D 2022-11-23T16:08:49.765 +C Add\soptional\szSchema\sargument\sto\ssqlite3_js_db_export(). +D 2022-11-23T21:03:22.603 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,12 +503,12 @@ F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b -F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 +F ext/wasm/api/sqlite3-api-prologue.js fa28d080a28bb936348156f7a298e0a001be088fb439905ff2dd85146f4fd2a0 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 +F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -R 6b5395a59674684ae7409c1a67752bd1 -T +closed 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -U dan -Z 08a51ae8157c87edcee2b45d299f4155 +P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 +R f8ada562338905eb066689b281095c3e +U stephan +Z 3d9c890d275324696db830c3a91a1a42 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c19a5133e..b3bb5e9e42 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 \ No newline at end of file +9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa \ No newline at end of file From f1ce4f40c7a01fd272662ab42d24b1f3ba747069 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 21:09:51 +0000 Subject: [PATCH 28/72] Minor JS doc updates. FossilOrigin-Name: 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 --- ext/wasm/api/sqlite3-api-prologue.js | 14 +++++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index f9ad5b5127..1cddc8eaf6 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1307,12 +1307,16 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( }; /** - Serializes the given `sqlite3*` pointer to a Uint8Array, as per - sqlite3_serialize(). On success it returns a Uint8Array. On - error it throws with a description of the problem. + A convenience wrapper around sqlite3_serialize() which serializes + the given `sqlite3*` pointer to a Uint8Array. - schema is the schema to serialize. It may be a WASM C-string - pointer or a JS string. If it is falsy, it defaults to "main". + On success it returns a Uint8Array. If the schema is empty, an + empty array is returned. + + `schema` is the schema to serialize. It may be a WASM C-string + pointer or a JS string. If it is falsy, it defaults to `"main"`. + + On error it throws with a description of the problem. */ capi.sqlite3_js_db_export = function(pDb, schema=0){ if(!pDb) toss3('Invalid sqlite3* argument.'); diff --git a/manifest b/manifest index 12cb76e8ab..c042b8b5eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soptional\szSchema\sargument\sto\ssqlite3_js_db_export(). -D 2022-11-23T21:03:22.603 +C Minor\sJS\sdoc\supdates. +D 2022-11-23T21:09:51.213 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b -F ext/wasm/api/sqlite3-api-prologue.js fa28d080a28bb936348156f7a298e0a001be088fb439905ff2dd85146f4fd2a0 +F ext/wasm/api/sqlite3-api-prologue.js 58caf45e4cc0deec9bdddb051ce17408686f2bc21ea7b3ee0b4c4deaba532605 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 -R f8ada562338905eb066689b281095c3e +P 9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa +R 8f8d491e9ca84b75fecb920dbf623854 U stephan -Z 3d9c890d275324696db830c3a91a1a42 +Z a1e4b87d56bdef1be072540c12f0e1a7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b3bb5e9e42..c0426c7035 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa \ No newline at end of file +27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 \ No newline at end of file From 9b80cb5f39af66075762057df127ffbbb2c6ef2a Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 01:40:20 +0000 Subject: [PATCH 29/72] Add explanatory comment to the new optimization. And add a test case. FossilOrigin-Name: e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 --- manifest | 13 ++++--- manifest.uuid | 2 +- src/select.c | 5 +++ test/tkt-99378177930f87bd.test | 69 ++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 test/tkt-99378177930f87bd.test diff --git a/manifest b/manifest index 5a5586e90e..9476f82d3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Aggregates\swith\sGROUP\sBY\snow\smake\suse\sof\sexpressions\son\sindexes.\s\sThis\scode\nworks\sand\sgets\sthe\scorrect\sanswer\sfor\sthe\stest\scase\sin\sthe\sticket.\s\sLots\smore\ntesting\sand\sdocumentation\sis\sneeded,\showever. -D 2022-11-23T18:51:04.363 +C Add\sexplanatory\scomment\sto\sthe\snew\soptimization.\s\sAnd\sadd\sa\stest\scase. +D 2022-11-24T01:40:20.262 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c f253214cbd2458f744b95fc6bf8dee103cecf4b051f620fd5f62b43dbab92ec1 +F src/select.c 7fcbbc0db92d0082e3a10b4492457932d8589e9027e843ac2b972a4ba0233136 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1628,6 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 +F test/tkt-99378177930f87bd.test f33bf2e038025941eb2f5495db4d4e3f5a614cdf6c44e0fe4d420d41197330c1 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2059,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 -R 2344df85f68d9987b55d50d38bac47c1 +P 8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f +R 9274a8d11a500d33c0e780bbd7f350f0 U drh -Z 4c924f435822db601e489fbe4dcd4a9e +Z b2115b9d996407ff47bc262d3bc9ad3e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 56c84edce5..cdcf19b4bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f \ No newline at end of file +e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 \ No newline at end of file diff --git a/src/select.c b/src/select.c index dcb878dde9..dcf899e4bb 100644 --- a/src/select.c +++ b/src/select.c @@ -7703,6 +7703,11 @@ int sqlite3Select( pAggInfo->useSortingIdx = 1; } + /* If there entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ if( pParse->pIdxEpr ){ aggregateConvertIndexedExprRefToColumn(pAggInfo); #if TREETRACE_ENABLED diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test new file mode 100644 index 0000000000..aad0a6dc21 --- /dev/null +++ b/test/tkt-99378177930f87bd.test @@ -0,0 +1,69 @@ +# 2022-11-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests to verify that the enhancement +# request documented by ticket 99378177930f87bd is working. +# +# The enhancement is that if an aggregate query with a GROUP BY clause +# uses subexpressions in the arguments to aggregate functions that are +# also columns of an index, then the values are pulled from the index +# rather than being recomputed. This has the potential to make some +# indexed queries works as if the index were covering. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test tkt-99378-100 { + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} {} +do_execsql_test tkt-99378-110 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} + +# The proof that the index on the expression is being used is in the +# fact that the byte code contains no "Function" opcodes. In other words, +# the ->> operator (which is implemented by a function) is never invoked. +# Instead, the b->>'x' value is pulled out of the index. +# +do_execsql_test tkt-99378-120 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + +finish_test From f46091d73fb059de729fe780a8d2eb4ff8056ddc Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 02:35:03 +0000 Subject: [PATCH 30/72] Add sqlite3.oo1.DB.prototype.checkRc() and tests for both that method and its class-level counterpart. FossilOrigin-Name: f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 --- ext/wasm/api/sqlite3-api-oo1.js | 27 +++++++++++++++++++-------- ext/wasm/tester1.c-pp.js | 10 ++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 02ce9c0ced..fc0f7372b8 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -55,12 +55,13 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(sqliteResultCode){ if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; toss3( - "sqlite result code",sqliteResultCode+":", + "sqlite3 result code",sqliteResultCode+":", (dbPtr ? capi.sqlite3_errmsg(dbPtr) : capi.sqlite3_errstr(sqliteResultCode)) ); } + return arguments[0]; }; /** @@ -462,14 +463,16 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Expects to be given a DB instance or an `sqlite3*` pointer (may be null) and an sqlite3 API result code. If the result code is not falsy, this function throws an SQLite3Error with an error - message from sqlite3_errmsg(), using dbPtr as the db handle, or - sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a - non-error code like SQLITE_ROW or SQLITE_DONE, it will still - throw but the error string might be "Not an error." The various - non-0 non-error codes need to be checked for in - client code where they are expected. + message from sqlite3_errmsg(), using db (or, if db is-a DB, + db.pointer) as the db handle, or sqlite3_errstr() if db is + falsy. Note that if it's passed a non-error code like SQLITE_ROW + or SQLITE_DONE, it will still throw but the error string might be + "Not an error." The various non-0 non-error codes need to be + checked for in client code where they are expected. + + If it does not throw, it returns its first argument. */ - DB.checkRc = checkSqlite3Rc; + DB.checkRc = (db,resultCode)=>checkSqlite3Rc(db,resultCode); DB.prototype = { /** Returns true if this db handle is open, else false. */ @@ -1130,6 +1133,14 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1"); throw e; } + }, + + /** + A convenience form of DB.checkRc(this,resultCode). If it does + not throw, it returns this object. + */ + checkRc: function(resultCode){ + return DB.checkRc(this, resultCode); } }/*DB.prototype*/; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 2a5da8407c..5bb5959396 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1163,6 +1163,16 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) .assert(dbFile === db.dbFilename()) .assert(!db.dbFilename('nope')); + //Sanity check DB.checkRc()... + let ex; + try{db.checkRc(rc)} + catch(e){ex = e} + T.assert(ex instanceof sqlite3.SQLite3Error) + .assert(0===ex.message.indexOf("sqlite3 result code")) + .assert(ex.message.indexOf("Invalid SQL")>0); + T.assert(db === db.checkRc(0)) + .assert(db === sqlite3.oo1.DB.checkRc(db,0)) + .assert(null === sqlite3.oo1.DB.checkRc(null,0)) }) //////////////////////////////////////////////////////////////////// diff --git a/manifest b/manifest index c042b8b5eb..a47d3eccc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sJS\sdoc\supdates. -D 2022-11-23T21:09:51.213 +C Add\ssqlite3.oo1.DB.prototype.checkRc()\sand\stests\sfor\sboth\sthat\smethod\sand\sits\sclass-level\scounterpart. +D 2022-11-24T02:35:03.924 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,7 +501,7 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 -F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed +F ext/wasm/api/sqlite3-api-oo1.js dec6c14994317ad0011714890426cdc211f4eab451c9766ea88c7ac4f535287e F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b F ext/wasm/api/sqlite3-api-prologue.js 58caf45e4cc0deec9bdddb051ce17408686f2bc21ea7b3ee0b4c4deaba532605 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f @@ -551,7 +551,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d +F ext/wasm/tester1.c-pp.js 3b91f192c159088004fba6fe3441edea58421a8b88bccf3dd20978a077648d19 F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa -R 8f8d491e9ca84b75fecb920dbf623854 +P 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 +R 25ba86c410fa1767dbcf638c18b7bc32 U stephan -Z a1e4b87d56bdef1be072540c12f0e1a7 +Z 6ecc8a9a17b94657e9cbb62c36aba3fc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0426c7035..1c21bd8831 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 \ No newline at end of file +f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 \ No newline at end of file From 9b9017def4edc224e0f4f773b775cfa575cf198b Mon Sep 17 00:00:00 2001 From: larrybr Date: Thu, 24 Nov 2022 02:59:33 +0000 Subject: [PATCH 31/72] Speed up base64 conversions, and add test with more data for the baseNN conversion to grind. FossilOrigin-Name: 6c84ae4ba83713c751fddff8be41686bbcb525ac8135e1520436c62d0bc23d2c --- ext/misc/base64.c | 70 +++++++++++++++++++++++++++-------------------- manifest | 14 +++++----- manifest.uuid | 2 +- test/basexx1.test | 14 ++++++++++ 4 files changed, 62 insertions(+), 38 deletions(-) mode change 100644 => 100755 ext/misc/base64.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c old mode 100644 new mode 100755 index 6e3a6b5cb7..7bb7683d1f --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -92,42 +92,46 @@ static const char b64Numerals[64+1] #define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) #define IS_BX_WS(bdp) ((bdp)==WS) #define IS_BX_PAD(bdp) ((bdp)==PC) -#define BX_NUMERAL(dv) (b64Numerals[dv]) +#define BX_NUMERAL(dv) (b64Numerals[(ubyte)(dv)]) /* Width of base64 lines. Should be an integer multiple of 4. */ #define B64_DARK_MAX 72 -/* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to B64_DARK_MAX -** or to terminate the last group (to aid concatenation.) +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. */ -static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ +static char* toBase64( ubyte *pIn, int nbIn, char *pOut ){ int nCol = 0; - *pOut = 0; - while( nbIn > 0 ){ - static signed char ncio[] = { 0, 2, 3, 4 }; - int nbi = (nbIn > 3)? 3 : nbIn; - signed char nc; - int nbe; - unsigned long qv = (ubyte)*pIn++; - for( nbe=1; nbe<3; ++nbe ){ - ubyte b = (nbe= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ - char ce = (nbe>= 6; pOut[nbe] = ce; } pOut += 4; - if( pSep && ((nCol += 4)>=B64_DARK_MAX || nbIn<=0) ){ - char *p = pSep; - while( *p ) *pOut++ = *p++; - nCol = 0; - } - *pOut = 0; + *pOut++ = '\n'; } + *pOut = 0; return pOut; } @@ -157,11 +161,12 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ ubyte bdp = BX_DV_PROTO(c); switch( bdp ){ case ND: - /* Treat non-digits as pad, but they terminate decode too. */ + /* Treat dark non-digits as pad, but they terminate decode too. */ ncIn = 0; /* fall thru */ case WS: - /* Treat whitespace as pad */ + /* Treat whitespace as pad and terminate this group.*/ + nti = nac; /* fall thru */ case PC: bdp = 0; @@ -172,10 +177,15 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; } } - nti = 2; - while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nti--))&0xff; + switch( nbo ){ + case 3: + pOut[2] = (qv) & 0xff; + case 2: + pOut[1] = (qv>>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; } + pOut += nbo; } return pOut; } @@ -200,7 +210,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; bBuf = (ubyte*)sqlite3_value_blob(av[0]); - nc = (int)(toBase64(bBuf, nb, cBuf, "\n") - cBuf); + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; case SQLITE_TEXT: diff --git a/manifest b/manifest index 6d54a98758..56f984f662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smore\sbaseNN\stests,\sget\soversize\serror\strapping\sworking,\sand\ssync\sw/trunk -D 2022-11-22T22:46:41.866 +C Speed\sup\sbase64\sconversions,\sand\sadd\stest\swith\smore\sdata\sfor\sthe\sbaseNN\sconversion\sto\sgrind. +D 2022-11-24T02:59:33.748 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,7 +289,7 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 0472f388e1a6c168912a363dfbe88d2d588325fba7ce6101f726d01b54fe6d3b +F ext/misc/base64.c 6333194e5c2e85b0748116ad4004bf3e070347cc09984aaa8d462fb3fc0566b6 x F ext/misc/base85.c 9005549904fc06ec2f3ff96970709f92f76e2d9ec2b785553ac32908ddc1baa0 F ext/misc/basexx.c 678dcc83894f78c26fd3662b322886777cc26bf2b40809236cd2abdad532a33c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a @@ -816,7 +816,7 @@ F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f310fd3b24a491b6b77bccdf14923b85d6ebcce751068c180d93a6b8ff854399 -F test/basexx1.test 9b12557d2b5bd017f9f8a8698239438ced5899c3ee55d9a549f74d90b16e51a6 +F test/basexx1.test d8a50f0744b93dca656625597bcd3499ff4b9a4ea2a82432b119b7d46e3e0c08 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c F test/bestindex1.test 856a453dff8c68b4568601eed5a8b5e20b4763af9229f3947c215729ed878db0 F test/bestindex2.test 394ff8fbf34703391247116d6a44e1c50ee7282236ee77909044573cefc37bc0 @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff67460e1a3d21c9ca7cbd171fbc5e6cbdb3797de359887f851776b73b732fdf 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R 36565c16629af64673142ebb24820a85 +P 03819e9368fd9f78f351147a1dc865743f9634893e43a9d1e3d7cbaf4c966069 +R f94c977ea1969bc60efbcf2158a80ba4 U larrybr -Z 3e47c0e49ec42d7b0a58333b563577b4 +Z 312bc23355f0a0c3b5b7524b3c0fa709 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 825fd93e96..11257f9dab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03819e9368fd9f78f351147a1dc865743f9634893e43a9d1e3d7cbaf4c966069 \ No newline at end of file +6c84ae4ba83713c751fddff8be41686bbcb525ac8135e1520436c62d0bc23d2c \ No newline at end of file diff --git a/test/basexx1.test b/test/basexx1.test index 8855a78634..947a5678f3 100644 --- a/test/basexx1.test +++ b/test/basexx1.test @@ -138,4 +138,18 @@ do_catchsql_test 116 { SELECT is_base85(x'00'); } {1 {is_base85 accepts only text or NULL}} +# Round-trip many bigger random blobs. + +do_execsql_test 117 { + CREATE TABLE bs(b blob, num); + INSERT INTO bs SELECT randomblob(4000 + n%3), n + FROM ( + WITH RECURSIVE seq(n) AS ( + VALUES(1) UNION ALL SELECT n+1 + FROM seq WHERE n<100 + ) SELECT n FROM seq); + SELECT num FROM bs WHERE base64(base64(b))!=b; + SELECT num FROM bs WHERE base85(base85(b))!=b; +} {} + finish_test From b669bb5e2bd130b2a8d8a87304449cfb3b27f9d9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 13:19:25 +0000 Subject: [PATCH 32/72] New test cases. Fix the logic so that it works for GROUP BY aggregates that do not require sorting. FossilOrigin-Name: ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef --- manifest | 14 +++--- manifest.uuid | 2 +- src/expr.c | 5 ++ test/tkt-99378177930f87bd.test | 85 ++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 19b2b20356..7f64f61129 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sagg-with-indexed-expr\sbranch\sto\nsimplify\sdiffs. -D 2022-11-24T01:41:44.524 +C New\stest\scases.\s\sFix\sthe\slogic\sso\sthat\sit\sworks\sfor\sGROUP\sBY\saggregates\nthat\sdo\snot\srequire\ssorting. +D 2022-11-24T13:19:25.140 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 141af8139174010ab37591df6234a647ecc38b41f72ac6e2c128ebcf167e0bc0 +F src/expr.c 44a7f638eebe915ca7522464f4753eb767bf37139c0010017eae1c60cb02345f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -1628,7 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 -F test/tkt-99378177930f87bd.test f33bf2e038025941eb2f5495db4d4e3f5a614cdf6c44e0fe4d420d41197330c1 +F test/tkt-99378177930f87bd.test 0f932e85fa1d41f30532cb7be9718d82e491e953123b8c4c85cf025f36ffe34b F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 -R beb6109a205ba56e52e9073d9db7b007 +P 38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 +R fe9857b5384581f396eba27025a5561f U drh -Z 9965e8c3c2c486ad5a3aae007bdf6965 +Z d9a080fe6c44b0ca440456485b55fa7a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6fe2b265a2..016d1fad45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 \ No newline at end of file +ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c34a28dc47..2e909dd8ca 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4145,6 +4145,11 @@ expr_code_doover: } } return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ /* no break */ deliberate_fall_through diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index aad0a6dc21..b21be3f67f 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -66,4 +66,89 @@ do_execsql_test tkt-99378-120 { GROUP BY a; } {~/Function/} + +do_execsql_test tkt-99378-130 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-140 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} {~/Function/} + +do_execsql_test tkt-99378-200 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(a, d, b->>'x', c); +} +do_execsql_test tkt-99378-210 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-220 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} +do_execsql_test tkt-99378-230 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-240 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + + + + finish_test From c25f5ea6e838dabbd0721947a432c4d9717bda74 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 15:04:23 +0000 Subject: [PATCH 33/72] Add NEVER() and ALWAYS() macros on branches that are believed to be unreachable. FossilOrigin-Name: 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/select.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f64f61129..c36f686e3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases.\s\sFix\sthe\slogic\sso\sthat\sit\sworks\sfor\sGROUP\sBY\saggregates\nthat\sdo\snot\srequire\ssorting. -D 2022-11-24T13:19:25.140 +C Add\sNEVER()\sand\sALWAYS()\smacros\son\sbranches\sthat\sare\sbelieved\sto\sbe\nunreachable. +D 2022-11-24T15:04:23.564 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 44a7f638eebe915ca7522464f4753eb767bf37139c0010017eae1c60cb02345f +F src/expr.c 07d4a0f36cd0fd52862ed25dffd4a746c4e12b6c9ceda3f87772ae373fb436a5 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7fcbbc0db92d0082e3a10b4492457932d8589e9027e843ac2b972a4ba0233136 +F src/select.c 6df5dd2a0434f0921f82cd23449e75138f05435025aed37ad96f344a7a31dcdb F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 -R fe9857b5384581f396eba27025a5561f +P ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef +R eb963117dcb4b03d7c27aa3277dea1bb U drh -Z d9a080fe6c44b0ca440456485b55fa7a +Z 65f0cab32e4b61b60d1aa8b06e205831 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 016d1fad45..17ef3a10aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef \ No newline at end of file +3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 2e909dd8ca..dc918e0bd6 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6335,14 +6335,14 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ int iDataCur = pIEpr->iDataCur; if( iDataCur<0 ) continue; - if( pParse->iSelfTab ){ + if( NEVER(pParse->iSelfTab) ){ if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; iDataCur = -1; } if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; } if( pIEpr==0 ) break; - if( !ExprUseYTab(pExpr) ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; /* If we reach this point, it means that expression pExpr can be ** translated into a reference to an index column as described by diff --git a/src/select.c b/src/select.c index dcf899e4bb..e9b6296713 100644 --- a/src/select.c +++ b/src/select.c @@ -6282,7 +6282,7 @@ static void optimizeAggregateUseOfIndexedExpr( NameContext *pNC /* Name context used to resolve agg-func args */ ){ pAggInfo->nColumn = pAggInfo->nAccumulator; - if( pAggInfo->nSortingColumn>0 ){ + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ if( pAggInfo->nColumn==0 ){ pAggInfo->nSortingColumn = 0; }else{ From 647b0dd12d23c084eaa037abc99ce0fb097d3b51 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Nov 2022 15:32:00 +0000 Subject: [PATCH 34/72] Update multiplex3.test to account for the fact that the multiplexor xDelete method may return an error even if it manages to delete the first chunk of a file. FossilOrigin-Name: 1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/multiplex3.test | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a47d3eccc2..f025974fe5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssqlite3.oo1.DB.prototype.checkRc()\sand\stests\sfor\sboth\sthat\smethod\sand\sits\sclass-level\scounterpart. -D 2022-11-24T02:35:03.924 +C Update\smultiplex3.test\sto\saccount\sfor\sthe\sfact\sthat\sthe\smultiplexor\sxDelete\smethod\smay\sreturn\san\serror\seven\sif\sit\smanages\sto\sdelete\sthe\sfirst\schunk\sof\sa\sfile. +D 2022-11-24T15:32:00.589 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1340,7 +1340,7 @@ F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a -F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 +F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 F test/mutex1.test 177db2e4edb530f2ff21edc52ac79a412dbe63e4c47c3ae9504d3fb4f1ce81fa F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 -R 25ba86c410fa1767dbcf638c18b7bc32 -U stephan -Z 6ecc8a9a17b94657e9cbb62c36aba3fc +P f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 +R f39101951334943bf67c026388719509 +U dan +Z db7c150da52ab80c381cc3a21444c8d8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1c21bd8831..92bbfc67cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 \ No newline at end of file +1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 \ No newline at end of file diff --git a/test/multiplex3.test b/test/multiplex3.test index c1e741acdb..3188350163 100644 --- a/test/multiplex3.test +++ b/test/multiplex3.test @@ -82,6 +82,8 @@ do_faultsim_test 1 -prep { multiplex_restore_db sqlite3 db file:test.db?8_3_names=1 sqlite3_multiplex_control db main chunk_size [expr 256*1024] + execsql { PRAGMA journal_mode = truncate } + execsql { PRAGMA synchronous = off } } -body { execsql { UPDATE t1 SET a=randomblob(12), b=randomblob(1500) WHERE (rowid%32)=0 From 6b86b33793421dc4136114fd6a7bd4f8e46f84c5 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 16:03:49 +0000 Subject: [PATCH 35/72] JS documentation cleanups. No code changes. FossilOrigin-Name: 18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 --- ext/wasm/api/sqlite3-api-prologue.js | 102 ++++++--------------------- manifest | 14 ++-- manifest.uuid | 2 +- 3 files changed, 29 insertions(+), 89 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 1cddc8eaf6..726f47e18d 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -11,76 +11,18 @@ *********************************************************************** This file is intended to be combined at build-time with other - related code, most notably a header and footer which wraps this whole - file into an Emscripten Module.postRun() handler which has a parameter - named "Module" (the Emscripten Module object). The exact requirements, - conventions, and build process are very much under construction and - will be (re)documented once they've stopped fluctuating so much. + related code, most notably a header and footer which wraps this + whole file into an Emscripten Module.postRun() handler which has a + parameter named "Module" (the Emscripten Module object). The sqlite3 + JS API has no hard requirements on Emscripten, and does not expose + any Emscripten APIs to clients. It is structured such that its build + can be tweaked to include it in arbitrary WASM environments which + supply the necessary underlying features (e.g. a POSIX file I/O + layer). - Project home page: https://sqlite.org + Main project home page: https://sqlite.org Documentation home page: https://sqlite.org/wasm - - Specific goals of this subproject: - - - Except where noted in the non-goals, provide a more-or-less - feature-complete wrapper to the sqlite3 C API, insofar as WASM - feature parity with C allows for. In fact, provide at least 4 - APIs... - - 1) 1-to-1 bindings as exported from WASM, with no automatic - type conversions between JS and C. - - 2) A binding of (1) which provides certain JS/C type conversions - to greatly simplify its use. - - 3) A higher-level API, more akin to sql.js and node.js-style - implementations. This one speaks directly to the low-level - API. This API must be used from the same thread as the - low-level API. - - 4) A second higher-level API which speaks to the previous APIs via - worker messages. This one is intended for use in the main - thread, with the lower-level APIs installed in a Worker thread, - and talking to them via Worker messages. Because Workers are - asynchronouns and have only a single message channel, some - acrobatics are needed here to feed async work results back to - the client (as we cannot simply pass around callbacks between - the main and Worker threads). - - - Insofar as possible, support client-side storage using JS - filesystem APIs. As of this writing, such things are still very - much under development. - - Specific non-goals of this project: - - - As WASM is a web-centric technology and UTF-8 is the King of - Encodings in that realm, there are no currently plans to support - the UTF16-related sqlite3 APIs. They would add a complication to - the bindings for no appreciable benefit. Though web-related - implementation details take priority, and the JavaScript - components of the API specifically focus on browser clients, the - lower-level WASM module "should" work in non-web WASM - environments. - - - Supporting old or niche-market platforms. WASM is built for a - modern web and requires modern platforms. - - - Though scalar User-Defined Functions (UDFs) may be created in - JavaScript, there are currently no plans to add support for - aggregate and window functions. - - Attribution: - - This project is endebted to the work of sql.js: - - https://github.com/sql-js/sql.js - - sql.js was an essential stepping stone in this code's development as - it demonstrated how to handle some of the WASM-related voodoo (like - handling pointers-to-pointers and adding JS implementations of - C-bound callback functions). These APIs have a considerably - different shape than sql.js's, however. */ /** @@ -90,6 +32,10 @@ for the current environment, and then optionally be removed from the global object using `delete self.sqlite3ApiBootstrap`. + This function is not intended for client-level use. It is intended + for use in creating bundles configured for specific WASM + environments. + This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so that it can be used without any _direct_ dependency on @@ -126,11 +72,11 @@ environment. Defaults to `"free"`. - `wasmfsOpfsDir`[^1]: if the environment supports persistent - storage, this directory names the "mount point" for that - directory. It must be prefixed by `/` and may contain only a - single directory-name part. Using the root directory name is not - supported by any current persistent backend. This setting is - only used in WASMFS-enabled builds. + storage using OPFS-over-WASMFS , this directory names the "mount + point" for that directory. It must be prefixed by `/` and may + contain only a single directory-name part. Using the root + directory name is not supported by any current persistent + backend. This setting is only used in WASMFS-enabled builds. [^1] = This property may optionally be a function, in which case this @@ -191,11 +137,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( const capi = Object.create(null); /** Holds state which are specific to the WASM-related - infrastructure and glue code. It is not expected that client - code will normally need these, but they're exposed here in case - it does. These APIs are _not_ to be considered an - official/stable part of the sqlite3 WASM API. They may change - as the developers' experience suggests appropriate changes. + infrastructure and glue code. Note that a number of members of this object are injected dynamically after the api object is fully constructed, so @@ -228,7 +170,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( result of sqlite3.capi.sqlite3_js_rc_str() or (if that returns falsy) a synthesized string which contains that integer. - - If passed 2 arguments and the 2nd is a object, it bevaves + - If passed 2 arguments and the 2nd is a object, it behaves like the Error(string,object) constructor except that the first argument is subject to the is-integer semantics from the previous point. @@ -686,9 +628,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** The WASM IR (Intermediate Representation) value for pointer-type values. It MUST refer to a value type of the - size described by this.ptrSizeof _or_ it may be any value - which ends in '*', which Emscripten's glue code internally - translates to i32. + size described by this.ptrSizeof. */ ptrIR: config.wasmPtrIR || "i32", /** diff --git a/manifest b/manifest index f025974fe5..a78a33a4bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\smultiplex3.test\sto\saccount\sfor\sthe\sfact\sthat\sthe\smultiplexor\sxDelete\smethod\smay\sreturn\san\serror\seven\sif\sit\smanages\sto\sdelete\sthe\sfirst\schunk\sof\sa\sfile. -D 2022-11-24T15:32:00.589 +C JS\sdocumentation\scleanups.\sNo\scode\schanges. +D 2022-11-24T16:03:49.104 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js dec6c14994317ad0011714890426cdc211f4eab451c9766ea88c7ac4f535287e F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b -F ext/wasm/api/sqlite3-api-prologue.js 58caf45e4cc0deec9bdddb051ce17408686f2bc21ea7b3ee0b4c4deaba532605 +F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 -R f39101951334943bf67c026388719509 -U dan -Z db7c150da52ab80c381cc3a21444c8d8 +P 1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 +R abd9b2d5004cf0e1068a5fc8037a67b4 +U stephan +Z 59d8b0c4ff23bb85893a1a0d379c6144 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 92bbfc67cc..7fa3372b8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 \ No newline at end of file +18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 \ No newline at end of file From df5d06d03eca407aa84f12fce477a0c210bc4375 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 17:53:09 +0000 Subject: [PATCH 36/72] More work on the OPFS concurrency testing app. FossilOrigin-Name: c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 --- ext/wasm/api/sqlite3-api-opfs.js | 3 +++ ext/wasm/api/sqlite3-opfs-async-proxy.js | 28 +++++----------------- ext/wasm/tests/opfs/concurrency/index.html | 11 +++++---- ext/wasm/tests/opfs/concurrency/test.js | 28 ++++++++++++++++++---- ext/wasm/tests/opfs/concurrency/worker.js | 8 +++---- manifest | 20 ++++++++-------- manifest.uuid | 2 +- 7 files changed, 54 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb8bf04e8..5537df89fb 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1171,6 +1171,9 @@ const installOpfsVfs = function callee(options){ //TODO to support fiddle and worker1 db upload: //opfsUtil.createFile = function(absName, content=undefined){...} + //We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this + //purpose but its interface and name are still under + //consideration. if(sqlite3.oo1){ opfsUtil.OpfsDb = function(...args){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index b14494a0c9..8a3db9c648 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -558,33 +558,17 @@ const vfsAsyncImpls = { (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite - does but it leads to horrible contention on journal files. */ + does but it leads to immediate contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ /* sqlite does not lock these files, so go ahead and grab an OPFS lock. - Regarding "immutable": that flag is not _really_ applicable - here. It's intended for use on read-only media. If, - however, a file is opened with that flag but changes later - (which can happen if we _don't_ grab a sync handle here) - then sqlite may misbehave. - - Regarding "nolock": ironically, the nolock flag forces us - to lock the file up front. "nolock" tells sqlite to _not_ - use its locking API, but OPFS requires a lock to perform - most of the operations performed in this file. If we don't - grab that lock up front, another handle could end up grabbing - it and mutating the database out from under our nolocked'd - handle. In the interest of preventing corruption, at the cost - of decreased concurrency, we have to lock it for the duration - of this file handle. - https://www.sqlite.org/uri.html */ - fh.xLock = "atOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; + fh.xLock = "xOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; await getSyncHandle(fh,'xOpen'); } __openFiles[fid] = fh; @@ -824,7 +808,7 @@ const waitLoop = async function f(){ to do other things. If this is too high (e.g. 500ms) then even two workers/tabs can easily run into locking errors. */ - const waitTime = 150; + const waitTime = 100; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index a082dfe997..e19f6a8da6 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -24,10 +24,13 @@

URL flags: pass a number of workers using - the workers=N URL flag and the worker work interval - as interval=N (milliseconds). Enable OPFS VFS - verbosity with verbose=1-3 (output goes to the - dev console). + the workers=N URL flag. Set the time between each + workload with interval=N (milliseconds). Set the + number of worker iterations with iterations=N. + Enable OPFS VFS verbosity with verbose=1-3 (output + goes to the dev console). Enable/disable "unlock ASAP" mode + (higher concurrency, lower speed) + with unlock-asap=0-1.

Achtung: if it does not start to do anything within a couple of seconds, check the dev console: Chrome often fails with "cannot allocate diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index cb9d4275be..044d343745 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -56,13 +56,16 @@ options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 3; + ) || 4; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 ) || 1; options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 - ) || 750; + ) || 1000; + options.iterations = ( + urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10 + ) || 10; options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; @@ -70,15 +73,25 @@ workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); }; - workers.loadedCount = 0; + workers.counts = {loaded: 0, passed: 0, failed: 0}; + const checkFinished = function(){ + if(workers.counts.passed + workers.counts.failed !== workers.length){ + return; + } + if(workers.counts.failed>0){ + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + }else{ + logCss('tests-pass',"All",workers.length,"workers finished."); + } + }; workers.onmessage = function(msg){ msg = msg.data; const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ case 'loaded': stdout(prefix,"loaded"); - if(++workers.loadedCount === workers.length){ - stdout("All workers loaded. Telling them to run..."); + if(++workers.counts.loaded === workers.length){ + stdout("All",workers.length,"workers loaded. Telling them to run..."); workers.post('run'); } break; @@ -86,10 +99,14 @@ case 'stderr': stderr(prefix,...msg.payload); break; case 'error': stderr(prefix,"ERROR:",...msg.payload); break; case 'finished': + ++workers.counts.passed; logCss('tests-pass',prefix,...msg.payload); + checkFinished(); break; case 'failed': + ++workers.counts.failed; logCss('tests-fail',prefix,"FAILED:",...msg.payload); + checkFinished(); break; default: logCss('error',"Unhandled message type:",msg); break; } @@ -100,6 +117,7 @@ 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + + '&iterations='+options.iterations + '&opfs-verbose='+options.opfsVerbose + '&opfs-unlock-asap='+options.unlockAsap ); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 91aa0fa6fa..95fefa1954 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -46,10 +46,9 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.oo1.DB({ + db = new sqlite3.opfs.OpfsDb({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c', - vfs: 'opfs' + flags: 'c' }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ @@ -59,7 +58,8 @@ self.sqlite3InitModule().then(async function(sqlite3){ ]); }); - const maxIterations = 10; + const maxIterations = + urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); diff --git a/manifest b/manifest index a6b0539bab..602f6943ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. -D 2022-11-23T20:49:08.427 +C More\swork\son\sthe\sOPFS\sconcurrency\stesting\sapp. +D 2022-11-24T17:53:09.937 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 +F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 +F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d -F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 -F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 +F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c +F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a +F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 -R ff8c3df82102d217683515f7779a95c4 +P 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 +R b591bd74ceba5bd36575b86f818be346 U stephan -Z 269769e48cdf286c8fb19455cdae3a52 +Z f64db814b4972f9f3353d092900f9ddf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bb0d09842b..11f565569e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file +c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 \ No newline at end of file From eb84c81c53b5b53f8ac259705a0702c57629e8bc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Nov 2022 17:58:55 +0000 Subject: [PATCH 37/72] Fix a test case in fts3expr4.test to account for different locales. FossilOrigin-Name: a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3expr4.test | 11 ++++++++++- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a78a33a4bb..b3aa3b73da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JS\sdocumentation\scleanups.\sNo\scode\schanges. -D 2022-11-24T16:03:49.104 +C Fix\sa\stest\scase\sin\sfts3expr4.test\sto\saccount\sfor\sdifferent\slocales. +D 2022-11-24T17:58:55.503 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1094,7 +1094,7 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test ebae205a7a89446c32583bcd492dcb817b9f6b31819bb4dde2583bb99c77e526 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8 -F test/fts3expr4.test f5b2832549f01b1f7f73389fa21d4b875499bc95bf7c8b36271844888c6a0938 +F test/fts3expr4.test 6c7675bbdbffe6ffc95e9e861500b8ac3f739c4d004ffda812f138eeb1b45529 F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1cf983e F test/fts3f.test 8c438d5e1cab526b0021988fb1dc70cf3597b006a33ffd6c955ee89929077fe3 F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45c5df45 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 -R abd9b2d5004cf0e1068a5fc8037a67b4 -U stephan -Z 59d8b0c4ff23bb85893a1a0d379c6144 +P 18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 +R e3089d3062e617db155d3fe3a44f9c32 +U dan +Z c38105809dd01fc34e6f6bf40716c164 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7fa3372b8a..bea5150fe5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 \ No newline at end of file +a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 \ No newline at end of file diff --git a/test/fts3expr4.test b/test/fts3expr4.test index 4a6bd293c4..b9227aef55 100644 --- a/test/fts3expr4.test +++ b/test/fts3expr4.test @@ -50,7 +50,16 @@ do_icu_expr_test 1.6 { "(x OR y)" } {PHRASE 3 0 ( x or y )} # is passed to the tokenizer. # do_icu_expr_test 1.7 {a:word} {PHRASE 0 0 word} -do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word} +# do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word} +do_test 1.8 { + set res [ + db one {SELECT fts3_exprtest('icu en_US', 'd:word', 'a', 'b', 'c')} + ] + expr { + $res=="PHRASE 3 0 d:word" || + $res=="AND {AND {PHRASE 3 0 d} {PHRASE 3 0 :}} {PHRASE 3 0 word}" + } +} 1 set sqlite_fts3_enable_parentheses 0 From 594357f2e100fb0b504a9fc1625d8f739c75b1d4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 18:45:56 +0000 Subject: [PATCH 38/72] Change a NEVER() into an assert(). FossilOrigin-Name: a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c36f686e3e..e69a4d81dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sNEVER()\sand\sALWAYS()\smacros\son\sbranches\sthat\sare\sbelieved\sto\sbe\nunreachable. -D 2022-11-24T15:04:23.564 +C Change\sa\sNEVER()\sinto\san\sassert(). +D 2022-11-24T18:45:56.655 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 07d4a0f36cd0fd52862ed25dffd4a746c4e12b6c9ceda3f87772ae373fb436a5 +F src/expr.c 4de4d488850fdf28ec2237401594af36713bec2fa739f6eda87e29ae8515fbb3 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef -R eb963117dcb4b03d7c27aa3277dea1bb +P 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 +R fd9fef17e074036cfa72866de0a2e639 U drh -Z 65f0cab32e4b61b60d1aa8b06e205831 +Z bd1dc748d429f9c2f3da3ec49c4b6323 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 17ef3a10aa..91406edc13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 \ No newline at end of file +a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index dc918e0bd6..f5587395a7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6330,15 +6330,12 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ default: { IndexedExpr *pIEpr; Expr tmp; + assert( pParse->iSelfTab==0 ); if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; if( pParse->pIdxEpr==0 ) break; for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ int iDataCur = pIEpr->iDataCur; if( iDataCur<0 ) continue; - if( NEVER(pParse->iSelfTab) ){ - if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; - iDataCur = -1; - } if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; } if( pIEpr==0 ) break; From e644f7c24bd97b24a0c1be9a82b11c13a316fafd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 19:02:40 +0000 Subject: [PATCH 39/72] Take care not to try to add comments to a TK_AGG_COLUMN opcode that does not have an associated Table object because it is a reference to an indexed expression. FossilOrigin-Name: 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e69a4d81dd..9d1dae173c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\sNEVER()\sinto\san\sassert(). -D 2022-11-24T18:45:56.655 +C Take\scare\snot\sto\stry\sto\sadd\scomments\sto\sa\sTK_AGG_COLUMN\sopcode\sthat\sdoes\nnot\shave\san\sassociated\sTable\sobject\sbecause\sit\sis\sa\sreference\sto\san\sindexed\nexpression. +D 2022-11-24T19:02:40.883 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 4de4d488850fdf28ec2237401594af36713bec2fa739f6eda87e29ae8515fbb3 +F src/expr.c af9e80644a53ba437851659ba753e8e049e32c40cd584eece3796357fab52e9e F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 -R fd9fef17e074036cfa72866de0a2e639 +P a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f +R 54c8f77b091e9c38ba79b231dfbd47a4 U drh -Z bd1dc748d429f9c2f3da3ec49c4b6323 +Z df0c8e1baf68d2353d611c7c44676419 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 91406edc13..979e5c34b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f \ No newline at end of file +5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f5587395a7..d7f8cfa917 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4135,9 +4135,11 @@ expr_code_doover: Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - if( pCol->iColumn<0 ){ + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( pTab!=0 ){ + }else{ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ From 8683c0928111707903654278e6c87634cbee5d8c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 23:35:27 +0000 Subject: [PATCH 40/72] Correctly deal with IF-NULL-ROW operators when dealing with indexed expressions in aggregates. FossilOrigin-Name: 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9d1dae173c..b0b6894258 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\scare\snot\sto\stry\sto\sadd\scomments\sto\sa\sTK_AGG_COLUMN\sopcode\sthat\sdoes\nnot\shave\san\sassociated\sTable\sobject\sbecause\sit\sis\sa\sreference\sto\san\sindexed\nexpression. -D 2022-11-24T19:02:40.883 +C Correctly\sdeal\swith\sIF-NULL-ROW\soperators\swhen\sdealing\swith\nindexed\sexpressions\sin\saggregates. +D 2022-11-24T23:35:27.975 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6df5dd2a0434f0921f82cd23449e75138f05435025aed37ad96f344a7a31dcdb +F src/select.c c1cf437e18022df196bfe6e1f34a5fc1e871706665a0588863df5082d2e26563 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f -R 54c8f77b091e9c38ba79b231dfbd47a4 +P 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b +R 9699b80135f783c01dc948dbd9568fdf U drh -Z df0c8e1baf68d2353d611c7c44676419 +Z b4118428f98b75714904c3dd7e4b5c40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 979e5c34b9..5c7c74dae1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b \ No newline at end of file +939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 \ No newline at end of file diff --git a/src/select.c b/src/select.c index e9b6296713..2e6ad45614 100644 --- a/src/select.c +++ b/src/select.c @@ -6318,6 +6318,7 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ if( pExpr->pAggInfo==0 ) return WRC_Continue; if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; pAggInfo = pExpr->pAggInfo; assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; From 7960da03466d96209f53defe088fc16916ba26fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:08:20 +0000 Subject: [PATCH 41/72] Improved comments. Add assert()s to verify that the AggInfo structure is unchanged after registers have been assigned. FossilOrigin-Name: 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 2 ++ src/select.c | 16 ++++++++++++++++ src/sqliteInt.h | 11 ++++++++--- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index b0b6894258..77a2dfc4d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\sdeal\swith\sIF-NULL-ROW\soperators\swhen\sdealing\swith\nindexed\sexpressions\sin\saggregates. -D 2022-11-24T23:35:27.975 +C Improved\scomments.\s\sAdd\sassert()s\sto\sverify\sthat\sthe\sAggInfo\sstructure\nis\sunchanged\safter\sregisters\shave\sbeen\sassigned. +D 2022-11-25T13:08:20.992 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c af9e80644a53ba437851659ba753e8e049e32c40cd584eece3796357fab52e9e +F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c c1cf437e18022df196bfe6e1f34a5fc1e871706665a0588863df5082d2e26563 +F src/select.c 6168029423615a01de361024c56261f2f39d85adcd6da06137010f7ceea759b4 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h f4917d663170308601d794cdff7b8cf9704c4bafe03d7e926a156be835e29f29 +F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b -R 9699b80135f783c01dc948dbd9568fdf +P 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 +R 85b4d1494a033b09f8257ada7ad96c3c U drh -Z b4118428f98b75714904c3dd7e4b5c40 +Z f7018e8b2be8d601c79b6c13f2f54a26 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c7c74dae1..07bc523f18 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 \ No newline at end of file +5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d7f8cfa917..d30ae97668 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6265,6 +6265,7 @@ static void findOrCreateAggInfoColumn( struct AggInfo_col *pCol; int k; + assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ if( pCol->iTable==pExpr->iTable @@ -6328,6 +6329,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo = pNC->uNC.pAggInfo; assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ default: { IndexedExpr *pIEpr; diff --git a/src/select.c b/src/select.c index 2e6ad45614..e76198772d 100644 --- a/src/select.c +++ b/src/select.c @@ -6253,6 +6253,7 @@ static void analyzeAggFuncArgs( ){ int i; assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); pNC->ncFlags |= NC_InAggFunc; for(i=0; inFunc; i++){ Expr *pExpr = pAggInfo->aFunc[i].pFExpr; @@ -6281,6 +6282,7 @@ static void optimizeAggregateUseOfIndexedExpr( AggInfo *pAggInfo, /* The aggregate info */ NameContext *pNC /* Name context used to resolve agg-func args */ ){ + assert( pAggInfo->iFirstReg==0 ); pAggInfo->nColumn = pAggInfo->nAccumulator; if( ALWAYS(pAggInfo->nSortingColumn>0) ){ if( pAggInfo->nColumn==0 ){ @@ -6348,6 +6350,18 @@ static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ ** Allocate a block of registers so that there is one register for each ** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first ** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After clling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ assert( pAggInfo!=0 ); @@ -6369,6 +6383,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; @@ -6436,6 +6451,7 @@ static void updateAccumulator( struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ecdb35fee..db65082bbc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2766,10 +2766,15 @@ struct AggInfo { }; /* -** Macros to compute aCol[] and aFunc[] register numbers: +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. */ -#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) -#define AggInfoFuncReg(A,I) ((A)->iFirstReg+(A)->nColumn+(I)) +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. From f5d0656f167b4b9b7730e0af0caefcb7893b4432 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:15:48 +0000 Subject: [PATCH 42/72] Improve the robustness of the updateAccumulator() routine against OOM. FossilOrigin-Name: d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 77a2dfc4d6..d326feb02b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments.\s\sAdd\sassert()s\sto\sverify\sthat\sthe\sAggInfo\sstructure\nis\sunchanged\safter\sregisters\shave\sbeen\sassigned. -D 2022-11-25T13:08:20.992 +C Improve\sthe\srobustness\sof\sthe\supdateAccumulator()\sroutine\sagainst\sOOM. +D 2022-11-25T13:15:48.547 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6168029423615a01de361024c56261f2f39d85adcd6da06137010f7ceea759b4 +F src/select.c 40a7d7b49b087789cdbf000e7b9328d3bbd29ee4ea585fad75e8b29fa567497c F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 -R 85b4d1494a033b09f8257ada7ad96c3c +P 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 +R d91874ddf96958a005f34b5d280b17b3 U drh -Z f7018e8b2be8d601c79b6c13f2f54a26 +Z b1915dcd60d8f6304cf98a15b81e844e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07bc523f18..7938fd8387 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 \ No newline at end of file +d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 \ No newline at end of file diff --git a/src/select.c b/src/select.c index e76198772d..215acd6a01 100644 --- a/src/select.c +++ b/src/select.c @@ -6430,8 +6430,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator @@ -6452,6 +6452,7 @@ static void updateAccumulator( struct AggInfo_col *pC; assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; From bedbb1bfabc0ba78206aa7897f874d168aa0af13 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:26:51 +0000 Subject: [PATCH 43/72] Remove a redundant assert() statement. FossilOrigin-Name: e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d326feb02b..728b14b688 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\srobustness\sof\sthe\supdateAccumulator()\sroutine\sagainst\sOOM. -D 2022-11-25T13:15:48.547 +C Remove\sa\sredundant\sassert()\sstatement. +D 2022-11-25T13:26:51.747 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 40a7d7b49b087789cdbf000e7b9328d3bbd29ee4ea585fad75e8b29fa567497c +F src/select.c f29c90b07067263139fa08f266860ba0877f7a1398775572c3996d3b67c2fe6e F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 -R d91874ddf96958a005f34b5d280b17b3 +P d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 +R b674915df2317c83a8051b3fae0cee5f U drh -Z b1915dcd60d8f6304cf98a15b81e844e +Z 8a9f5f62db508168a49cc0d82c6f6183 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7938fd8387..d164374f87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 \ No newline at end of file +e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 215acd6a01..187beac7fb 100644 --- a/src/select.c +++ b/src/select.c @@ -6387,7 +6387,6 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; - assert( pAggInfo->iFirstReg>0 ); if( pParse->nErr ) return; sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, pAggInfo->iFirstReg+nReg-1); From 23f61a4ba80695ee6e9d88e56294d0dae24d5a41 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 15:52:00 +0000 Subject: [PATCH 44/72] Add restriction (9) to the push-down optimization: If the subquery is a compound then all arms of the compound must have the same affinity. dbsqlfuzz 3a548de406a50e896c1bf7142692d35d339d697f. FossilOrigin-Name: 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f --- manifest | 16 +++++++------- manifest.uuid | 2 +- src/select.c | 53 ++++++++++++++++++++++++++++++++++------------ test/pushdown.test | 27 ++++++++++++++++++++++- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index b3aa3b73da..64d80c2f59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sin\sfts3expr4.test\sto\saccount\sfor\sdifferent\slocales. -D 2022-11-24T17:58:55.503 +C Add\srestriction\s(9)\sto\sthe\spush-down\soptimization:\s\sIf\sthe\ssubquery\sis\na\scompound\sthen\sall\sarms\sof\sthe\scompound\smust\shave\sthe\ssame\saffinity.\ndbsqlfuzz\s3a548de406a50e896c1bf7142692d35d339d697f. +D 2022-11-25T15:52:00.241 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 4c48373abb4e67129c36bc15d1f5a99a0dfd9534afeb539a2169a09ae91ccec9 +F src/select.c bafe6424e942aad558b2d4be8dbcf5e35ce427ce3cf66d7f0c0ac37c366b00c6 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1403,7 +1403,7 @@ F test/printf.test 390d0d7fcffc3c4ea3c1bb4cbb267444e32b33b048ae21895f23a291844fe F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc +F test/pushdown.test c69f0970ea17e0afc674b89741f60c172cb6f761d81665fc71015f674f0f66ba F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 -R e3089d3062e617db155d3fe3a44f9c32 -U dan -Z c38105809dd01fc34e6f6bf40716c164 +P a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 +R 9c0acbc69bb427be679e111e4ec839ec +U drh +Z ae1fb8eb841aaf444344209ddd2c7ea7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bea5150fe5..4e3e4aa9d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 \ No newline at end of file +1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f \ No newline at end of file diff --git a/src/select.c b/src/select.c index 4d5bde9a31..65a0a06c33 100644 --- a/src/select.c +++ b/src/select.c @@ -4050,6 +4050,34 @@ static ExprList *findLeftmostExprlist(Select *pSel){ return pSel->pEList; } +/* +** Return true if any of the result-set columns in the compound query +** have incompatible affinities on one or more arms of the compound. +*/ +static int compoundHasDifferentAffinities(Select *p){ + int ii; + ExprList *pList; + assert( p!=0 ); + assert( p->pEList!=0 ); + assert( p->pPrior!=0 ); + pList = p->pEList; + for(ii=0; iinExpr; ii++){ + char aff; + Select *pSub1; + assert( pList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pList->a[ii].pExpr); + for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 1; + } + } + } + return 0; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -4153,7 +4181,8 @@ static ExprList *findLeftmostExprlist(Select *pSel){ ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) ** (17h) The corresponding result set expressions in all arms of the -** compound must have the same affinity. +** compound must have the same affinity. (See restriction (9) +** on the push-down optimization.) ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -4372,19 +4401,7 @@ static int flattenSubquery( if( (p->selFlags & SF_Recursive) ) return 0; /* Restriction (17h) */ - for(ii=0; iipEList->nExpr; ii++){ - char aff; - assert( pSub->pEList->a[ii].pExpr!=0 ); - aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr); - for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){ - assert( pSub1->pEList!=0 ); - assert( pSub1->pEList->nExpr>ii ); - assert( pSub1->pEList->a[ii].pExpr!=0 ); - if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ - return 0; - } - } - } + if( compoundHasDifferentAffinities(pSub) ) return 0; if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; @@ -5036,6 +5053,11 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** But it is a lot of work to check that case for an obscure and ** minor optimization, so we omit it for now.) ** +** (9) If the subquery is a compound, then all arms of the compound must +** have the same affinity. (This is the same as restriction (17h) +** for query flattening.) +** +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -5061,6 +5083,9 @@ static int pushDownWhereTerms( if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ if( pSel->pWin ) return 0; /* restriction (6b) */ } + if( compoundHasDifferentAffinities(pSubq) ){ + return 0; /* restriction (9) */ + } }else{ if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; } diff --git a/test/pushdown.test b/test/pushdown.test index af5162495b..7c9b107841 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -86,6 +86,31 @@ do_test 2.2 { set L } {three} - +# 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f +# Disable the push-down optimization for compound subqueries if any +# arm of the compound has an incompatible affinity. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t0(c0 INT); + INSERT INTO t0 VALUES(0); + CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1_a VALUES(1,'one'); --,(4,'four'); + CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t1_b VALUES(2,'two'); --,(5,'five'); + CREATE VIEW v0 AS SELECT CAST(t0.c0 AS INTEGER) AS c0 FROM t0; + CREATE VIEW t1 AS SELECT a, b FROM t1_a UNION ALL SELECT c, 0 FROM t1_b; + SELECT t1.a, quote(t1.b), t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1; +} { + 1 'one' 0 + 2 '0' 0 +} +do_execsql_test 3.2 { + SELECT a, quote(b), cd FROM ( + SELECT t1.a, t1.b, t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1 + ) WHERE a=2 AND b='0' AND cd=0; +} { + 2 '0' 0 +} finish_test From 49d400b0912fe99f5d6c0bc9e361f132c4fa338e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 16:32:59 +0000 Subject: [PATCH 45/72] Fix #ifdefs so that restrictions (8) and (9) of the push-down optimization are still enforced even if compiled with SQLITE_OMIT_WINDOWFUNC. This fixes a bug introduced by check-in [346a3b12b861ce7b]. FossilOrigin-Name: 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/select.c | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 76e28676a0..8852b478ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Immprove\sthe\squery\splanner\ssuch\sthat\sit\sis\sable\sto\smake\suse\sof\sindexed\nexpressions\swithin\san\saggregate\squery\swith\sGROUP\sBY.\s\sThis\simplements\nenhancement\srequest\s[99378177930f87bd]. -D 2022-11-25T16:10:48.068 +C Fix\s#ifdefs\sso\sthat\srestrictions\s(8)\sand\s(9)\sof\sthe\spush-down\soptimization\nare\sstill\senforced\seven\sif\scompiled\swith\sSQLITE_OMIT_WINDOWFUNC.\s\sThis\nfixes\sa\sbug\sintroduced\sby\scheck-in\s[346a3b12b861ce7b]. +D 2022-11-25T16:32:59.610 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7dcab394efe24c6016c6ed7958d426bfcdef57f50c2520dcccda27d9df28fdb6 +F src/select.c e0a04b10826637f826e0f9edb9c6cebdc0031f4093cec2553030f39e0b751109 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 -R eb048b7302ca70762f9994708c895609 -T +closed e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 +P b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 +R ba0c31d84a1008c246f9efb185042449 U drh -Z 6fdb43be076439dd6a7f7f9247393df9 +Z 98b0ad17c9260369ac5243663e0dc610 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff7ccc702c..1de8ddd5c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 \ No newline at end of file +09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 067c98b4a3..bc9ae25ce8 100644 --- a/src/select.c +++ b/src/select.c @@ -5073,7 +5073,6 @@ static int pushDownWhereTerms( if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0; -#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pPrior ){ Select *pSel; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ @@ -5081,15 +5080,18 @@ static int pushDownWhereTerms( assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ +#endif } if( compoundHasDifferentAffinities(pSubq) ){ return 0; /* restriction (9) */ } }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; - } #endif + } #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make From 902ea8392536b729c4c4553613dae9f65a4c1bf3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 17:05:55 +0000 Subject: [PATCH 46/72] Relax restriction (8) on the push-down optimization so that it only applies if one or more columns uses a collating sequence other than BINARY. See [forum:/forumpost/3824ced748baa808|forum post 3824ced748baa808] and check-in [346a3b12b861ce7b]. FossilOrigin-Name: adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 30 +++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 8852b478ad..3a6ab9396c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\s#ifdefs\sso\sthat\srestrictions\s(8)\sand\s(9)\sof\sthe\spush-down\soptimization\nare\sstill\senforced\seven\sif\scompiled\swith\sSQLITE_OMIT_WINDOWFUNC.\s\sThis\nfixes\sa\sbug\sintroduced\sby\scheck-in\s[346a3b12b861ce7b]. -D 2022-11-25T16:32:59.610 +C Relax\srestriction\s(8)\son\sthe\spush-down\soptimization\sso\sthat\sit\sonly\sapplies\nif\sone\sor\smore\scolumns\suses\sa\scollating\ssequence\sother\sthan\sBINARY.\nSee\s[forum:/forumpost/3824ced748baa808|forum\spost\s3824ced748baa808]\sand\ncheck-in\s[346a3b12b861ce7b]. +D 2022-11-25T17:05:55.692 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c e0a04b10826637f826e0f9edb9c6cebdc0031f4093cec2553030f39e0b751109 +F src/select.c 934c37455762579dcef5ce37f6abf04facaa61ef04cde207ac170b28c4780113 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 -R ba0c31d84a1008c246f9efb185042449 +P 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 +R 051f7bebeb695c93dfdb45030ad5b532 U drh -Z 98b0ad17c9260369ac5243663e0dc610 +Z 0a4a218169578e69e7b7469760b002aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1de8ddd5c0..76e9e654c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 \ No newline at end of file +adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf \ No newline at end of file diff --git a/src/select.c b/src/select.c index bc9ae25ce8..2bedfb9da0 100644 --- a/src/select.c +++ b/src/select.c @@ -5046,12 +5046,9 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** be materialized. (This restriction is implemented in the calling ** routine.) ** -** (8) The subquery may not be a compound that uses UNION, INTERSECT, -** or EXCEPT. (We could, perhaps, relax this restriction to allow -** this case if none of the comparisons operators between left and -** right arms of the compound use a collation other than BINARY. -** But it is a lot of work to check that case for an obscure and -** minor optimization, so we omit it for now.) +** (8) If the subquery is a compound that uses UNION, INTERSECT, +** or EXCEPT, then all of the result set columns for all arms of +** the compound must use the BINARY collating sequence. ** ** (9) If the subquery is a compound, then all arms of the compound must ** have the same affinity. (This is the same as restriction (17h) @@ -5075,11 +5072,14 @@ static int pushDownWhereTerms( if( pSubq->pPrior ){ Select *pSel; + int notUnionAll = 0; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ u8 op = pSel->op; assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); - if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ + if( op!=TK_ALL && op!=TK_SELECT ){ + notUnionAll = 1; + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ #endif @@ -5087,6 +5087,22 @@ static int pushDownWhereTerms( if( compoundHasDifferentAffinities(pSubq) ){ return 0; /* restriction (9) */ } + if( notUnionAll ){ + /* If any of the compound arms are connected using UNION, INTERSECT, + ** or EXCEPT, then we must ensure that none of the columns use a + ** non-BINARY collating sequence. */ + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + int ii; + const ExprList *pList = pSel->pEList; + assert( pList!=0 ); + for(ii=0; iinExpr; ii++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr); + if( !sqlite3IsBinary(pColl) ){ + return 0; /* Restriction (8) */ + } + } + } + } }else{ #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; From 6e711869042b45d84fd6c8fe8243db3849287137 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 14:19:47 +0000 Subject: [PATCH 47/72] Experimental changes to help the query planner detect when an expression index is coverting. Works somewhat, but there are tests that fail. FossilOrigin-Name: 968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 --- manifest | 15 ++++++----- manifest.uuid | 2 +- src/where.c | 69 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 3a6ab9396c..a56bd219e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Relax\srestriction\s(8)\son\sthe\spush-down\soptimization\sso\sthat\sit\sonly\sapplies\nif\sone\sor\smore\scolumns\suses\sa\scollating\ssequence\sother\sthan\sBINARY.\nSee\s[forum:/forumpost/3824ced748baa808|forum\spost\s3824ced748baa808]\sand\ncheck-in\s[346a3b12b861ce7b]. -D 2022-11-25T17:05:55.692 +C Experimental\schanges\sto\shelp\sthe\squery\splanner\sdetect\swhen\san\sexpression\nindex\sis\scoverting.\s\sWorks\ssomewhat,\sbut\sthere\sare\stests\sthat\sfail. +D 2022-11-26T14:19:47.899 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba +F src/where.c 550388b75a29c2af2c2182e56e2a51bc6b92aef302eb072f177b3526394d6020 F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2060,8 +2060,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 -R 051f7bebeb695c93dfdb45030ad5b532 +P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf +R b02bf662fd2aa7710f36a7d19197692a +T *branch * covering-indexed-expr +T *sym-covering-indexed-expr * +T -sym-trunk * U drh -Z 0a4a218169578e69e7b7469760b002aa +Z e78db49817bfc2647feb4d11db6b69e2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 76e9e654c1..036305084f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf \ No newline at end of file +968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index ae3d513cae..5f35c1e791 100644 --- a/src/where.c +++ b/src/where.c @@ -3247,6 +3247,26 @@ static int whereUsablePartialIndex( return 0; } +/* +** pIdx is an index containing expressions. Check it see if any of the +** expressions in the index match the pExpr expression. +*/ +static int exprIsCoveredByIndex( + const Expr *pExpr, + const Index *pIdx, + int iTabCur +){ + int i; + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]==XN_EXPR + && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 + ){ + return 1; + } + } + return 0; +} + /* ** Structure passed to the whereIsCoveringIndex Walker callback. */ @@ -3256,7 +3276,7 @@ struct CoveringIndexCheck { }; /* -** Information passed in is pWalk->u.pCovIdxCk. Call is pCk. +** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. ** ** If the Expr node references the table with cursor pCk->iTabCur, then ** make sure that column is covered by the index pCk->pIdx. We know that @@ -3268,23 +3288,32 @@ struct CoveringIndexCheck { ** ** If this node does not disprove that the index can be a covering index, ** then just return WRC_Continue, to continue the search. +** +** If pCk->pIdx contains indexed expressions and one of those expressions +** matches pExpr, then prune the search. */ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ int i; /* Loop counter */ const Index *pIdx; /* The index of interest */ const i16 *aiColumn; /* Columns contained in the index */ u16 nColumn; /* Number of columns in the index */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue; - if( pExpr->iColumn<(BMS-1) ) return WRC_Continue; - if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; pIdx = pWalk->u.pCovIdxCk->pIdx; - aiColumn = pIdx->aiColumn; - nColumn = pIdx->nColumn; - for(i=0; iiColumn ) return WRC_Continue; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ + /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ + if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pWalk->eCode = 1; + return WRC_Abort; + }else if( pIdx->bHasExpr + && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + return WRC_Prune; } - pWalk->eCode = 1; - return WRC_Abort; + return WRC_Continue; } @@ -3315,14 +3344,16 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( ** if pIdx is covering. Assume it is not. */ return 1; } - for(i=0; inColumn; i++){ - if( pIdx->aiColumn[i]>=BMS-1 ) break; - } - if( i>=pIdx->nColumn ){ - /* pIdx does not index any columns greater than 62, but we know from - ** colMask that columns greater than 62 are used, so this is not a - ** covering index */ - return 1; + if( pIdx->bHasExpr==0 ){ + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 1; + } } ck.pIdx = pIdx; ck.iTabCur = iTabCur; @@ -3552,7 +3583,7 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT ){ + if( m==TOPBIT || pProbe->bHasExpr ){ m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; From 743c9ec4a5f3f00a85ad99c582b012c189a0d34f Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:12:02 +0000 Subject: [PATCH 48/72] Fix an incorrect assertion-fault in the "TREETRACE" debugging logic on select.c. The problem does not affect production builds. FossilOrigin-Name: f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f14f6424d5..a2d2f52121 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\sadd\sthe\sopfs-unlock-asap=1\sURI\sflag\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP\sinstead\sof\sduring\sVFS\sidle\stime.\sThis\simproves\sconcurrency\snotably\sin\sthe\stest\sapp\sbut\sbrings\sa\ssignificant\sperformance\spenalty\sin\sspeedtest1\s(roughly\s4x\sslowdown).\sThis\sis\snot\sthe\sfinal\sword\sin\sOPFS\sconcurrency,\sbut\sgets\sus\sa\sstep\sfurther. -D 2022-11-26T15:24:58.092 +C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:12:02.257 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 934c37455762579dcef5ce37f6abf04facaa61ef04cde207ac170b28c4780113 +F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 -R ef0be46e6ac8ee603582dd679fae7434 -T +closed c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 Closed\sby\sintegrate-merge. -U stephan -Z 3b8060cf3f2afe36aab99fa36bba89db +P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 +R 581898efb4aefcef8555315298dbd7f7 +U drh +Z f6493a79b992a3696ede08a261d497d4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e97840e0fc..d4e96fca06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 \ No newline at end of file +f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2bedfb9da0..10498310d4 100644 --- a/src/select.c +++ b/src/select.c @@ -6259,7 +6259,7 @@ static void printAggInfo(AggInfo *pAggInfo){ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d %s\n", ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, pCol->iSorterColumn, ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); From 217e77d24008f5bd813c719e4a08da93500a2218 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:48:56 +0000 Subject: [PATCH 49/72] Show which AggInfo.aCol an Expr node represents in the tree trace debugging logic. Does not affect production builds. FossilOrigin-Name: b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/treeview.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a2d2f52121..54ca01a2ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:12:02.257 +C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:48:56.633 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 +F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 -R 581898efb4aefcef8555315298dbd7f7 +P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e +R 77aba841340d0be9ea0f0036dfeb9ac7 U drh -Z f6493a79b992a3696ede08a261d497d4 +Z cf507f961f29b58691309417f6fc3d94 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d4e96fca06..4df4cac51e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file +b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 89a128dff6..9df16a12bf 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; From 9d34152bcf72c20ed260d2c124dfde292cf4d5f0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:52:38 +0000 Subject: [PATCH 50/72] Better reporting of when the WHERE clause analysis thinks that an index is covering. FossilOrigin-Name: 17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/treeview.c | 5 ++++- src/where.c | 9 ++++++++- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1e9ca78250..e31ab14280 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sinto\sthe\scovering-indexed-expr\sbranch. -D 2022-11-26T20:13:54.129 +C Better\sreporting\sof\swhen\sthe\sWHERE\sclause\sanalysis\sthinks\sthat\san\sindex\nis\scovering. +D 2022-11-26T20:52:38.117 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 +F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 550388b75a29c2af2c2182e56e2a51bc6b92aef302eb072f177b3526394d6020 +F src/where.c 1f64631ab2c94b3c11de00ad1d08be9d51ee73acd613c54c4367526f5e2ac241 F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e -R b11982e41348628501c0b91820bd7cd7 +P 9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e +R 1dffa8ce2c066e6a4b41c4abd6aac427 U drh -Z 5e6dcb1170e4740f335f77683444a169 +Z 2a27a64568a31340c97915e2beb3adb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 25771bfcb7..7f494549c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e \ No newline at end of file +17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 89a128dff6..9df16a12bf 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; diff --git a/src/where.c b/src/where.c index 5f35c1e791..6ab9230d83 100644 --- a/src/where.c +++ b/src/where.c @@ -3583,8 +3583,15 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT || pProbe->bHasExpr ){ + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol) ){ m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + WHERETRACE(0xff, + ("-> %s %s a covering index according to whereIsCoveringIndex()\n", + pProbe->zName, m==0 ? "is" : "is not")); + }else{ + WHERETRACE(0xff, + ("-> %s %s a covering index according to bitmasks\n", + pProbe->zName, m==0 ? "is" : "is not")); } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } From ca9309cc565da99c6a8e8dea84852b9c4ee478a4 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 27 Nov 2022 00:57:56 +0000 Subject: [PATCH 51/72] Document an OPFS API change in Chrome v108 which does not break our code but does change several formerly async methods to synchronous. No code changes. FossilOrigin-Name: e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 16 ++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8a3db9c648..8e60969bc6 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -29,6 +29,22 @@ This file represents an implementation detail of a larger piece of code, and not a public interface. Its details may change at any time and are not intended to be used by any client-level code. + + 2022-11-27: Chrome v108 changes some async methods to synchronous, as + documented at: + + https://developer.chrome.com/blog/sync-methods-for-accesshandles/ + + We cannot change to the sync forms at this point without breaking + clients who use Chrome v104-ish or higher. truncate(), getSize(), + flush(), and close() are now (as of v108) synchronous. Calling them + with an "await", as we have to for the async forms, is still legal + with the sync forms but is superfluous. Calling the async forms with + theFunc().then(...) is not compatible with the change to + synchronous, but we do do not use those APIs that way. i.e. we don't + _need_ to change anything for this, but at some point (after Chrome + versions (approximately) 104-107 are extinct) should change our + usage of those methods to remove the "await". */ "use strict"; const toss = function(...args){throw new Error(args.join(' '))}; diff --git a/manifest b/manifest index 54ca01a2ed..37ec0eef0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:48:56.633 +C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. +D 2022-11-27T00:57:56.426 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -506,7 +506,7 @@ F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18d F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 798383f6b46fd5dac122d6e35962d25b10401ddb825b5c66df1d21e6b1d8aacc F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e -R 77aba841340d0be9ea0f0036dfeb9ac7 -U drh -Z cf507f961f29b58691309417f6fc3d94 +P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 +R 4c0998f1bfd61979c1006462c1fac102 +U stephan +Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4df4cac51e..852ac7c5b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file +e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file From c1ca183abfeb54a5735f4f3bd4c0e1994645c3f7 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 28 Nov 2022 02:28:44 +0000 Subject: [PATCH 52/72] Conform CLI .trace arg handling to its help. FossilOrigin-Name: 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c.in | 2 +- test/shell4.test | 6 ++++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 37ec0eef0c..107bf9f15b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. -D 2022-11-27T00:57:56.426 +C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. +D 2022-11-28T02:28:44.889 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 -F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 +F src/shell.c.in 334b1661abb6453e4f5d1778ce9c80dddef83470867f1fba07e5ee400792adbe F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -1497,7 +1497,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test e4b4de56f454708e0747b52915135baa2cbfec4965406d6eaf02a4a5c22a9880 F test/shell2.test c536c2aab4852608f8a606262330797abc4d964a4c2c782a7760f54ea1f17a6a F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a -F test/shell4.test 7dc8a515705bc093d8ffe381670e8fa7a969661e8ed177c35c847e3c6dfc35e2 +F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 -R 4c0998f1bfd61979c1006462c1fac102 -U stephan -Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 +P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 +R ce11368d09e97cb20c99cf53926be781 +U larrybr +Z 2b24f85901c30dde8df5f7afa27a353b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 852ac7c5b3..2d0d9deb13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file +31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index e1f0ecfcd4..ab47dba8c9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10658,7 +10658,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); + p->traceOut = output_file_open(z, 0); } } if( p->traceOut==0 ){ diff --git a/test/shell4.test b/test/shell4.test index 068072202d..193dc8b69b 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -125,6 +125,12 @@ SELECT * FROM t1;}} do_test shell4-2.5 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;" } {0 {SELECT * FROM t1;}} +do_test shell4-2.6 { + catchcmd ":memory:" { +CREATE TABLE t1(x); +.trace --stmt stdout +SELECT * FROM t1;} +} {0 {SELECT * FROM t1;}} } do_test shell4-3.1 { From abbfa7aa17307f31011751072454850e6da4d435 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Nov 2022 13:47:27 +0000 Subject: [PATCH 53/72] Ensure the RETURNING clause is honoured when a row of a temp table is updated by an ON CONFLICT clause. FossilOrigin-Name: a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/trigger.c | 2 +- test/returning1.test | 24 ++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 107bf9f15b..91afe6c7c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. -D 2022-11-28T02:28:44.889 +C Ensure\sthe\sRETURNING\sclause\sis\shonoured\swhen\sa\srow\sof\sa\stemp\stable\sis\supdated\sby\san\sON\sCONFLICT\sclause. +D 2022-11-28T13:47:27.194 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -707,7 +707,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 -F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 +F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 @@ -1421,7 +1421,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/releasetest_data.tcl 0db8aee0c348090fd06da47020ab4ed8ec692e0723427b2f3947d4dfb806f3b0 F test/resetdb.test 8062cf10a09d8c048f8de7711e94571c38b38168db0e5877ba7561789e5eeb2b F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test c43b8370a351f77aec6d71f4a2cde59b849369ed1933261a2c2c69e23e34ff5e +F test/returning1.test 1366e04566cfe1a082d17b1e0f195ec64473c79374b3a5d4ae00c43d885dea31 F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 -R ce11368d09e97cb20c99cf53926be781 -U larrybr -Z 2b24f85901c30dde8df5f7afa27a353b +P 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c +R cab683454f2ed9db2c50c1dc1f9667b9 +U dan +Z f330cc4347042764fbf120efeed28ded # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d0d9deb13..0bd7375f33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file +a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa \ No newline at end of file diff --git a/src/trigger.c b/src/trigger.c index ca64d6145b..02d8540237 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -61,7 +61,7 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && pTrig->table && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - && pTrig->pTabSchema!=pTmpSchema + && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning) ){ pTrig->pNext = pList; pList = pTrig; diff --git a/test/returning1.test b/test/returning1.test index fb058a6438..92e10ee9f4 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -376,4 +376,28 @@ do_execsql_test 16.1 { SELECT * FROM t2; } {1 2 3 a b c} + +foreach {tn temp} { + 1 "" + 2 TEMP +} { + reset_db + do_execsql_test 17.$tn.0 " + CREATE $temp TABLE foo ( + fooid INTEGER PRIMARY KEY, + fooval INTEGER NOT NULL UNIQUE, + refcnt INTEGER NOT NULL DEFAULT 1 + ); + " + do_execsql_test 17.$tn.1 { + INSERT INTO foo (fooval) VALUES (17), (4711), (17) + ON CONFLICT DO + UPDATE SET refcnt = refcnt+1 + RETURNING fooid; + } { + 1 2 1 + } +} + + finish_test From 84606c973757cbb5a276b54c5bfad67943288722 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 15:23:53 +0000 Subject: [PATCH 54/72] Rework the covering index checking routine, whereIsCoveringIndex(), so that it can return a "maybe" result for aggregate queries where we are not exactly sure. The index is scored as if it is covering, but the main table is still opened. FossilOrigin-Name: b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a --- manifest | 14 +++---- manifest.uuid | 2 +- src/where.c | 103 ++++++++++++++++++++++++++++++++++--------------- src/whereInt.h | 1 + 4 files changed, 81 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index e31ab14280..f0f6a8af9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\sreporting\sof\swhen\sthe\sWHERE\sclause\sanalysis\sthinks\sthat\san\sindex\nis\scovering. -D 2022-11-26T20:52:38.117 +C Rework\sthe\scovering\sindex\schecking\sroutine,\swhereIsCoveringIndex(),\sso\sthat\nit\scan\sreturn\sa\s"maybe"\sresult\sfor\saggregate\squeries\swhere\swe\sare\snot\sexactly\nsure.\s\sThe\sindex\sis\sscored\sas\sif\sit\sis\scovering,\sbut\sthe\smain\stable\sis\nstill\sopened. +D 2022-11-28T15:23:53.775 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -728,8 +728,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 1f64631ab2c94b3c11de00ad1d08be9d51ee73acd613c54c4367526f5e2ac241 -F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f +F src/where.c 81422870d17fdf3880fdb9d3b3965e1aa5efb7a72c6933cec3f1bc8081c463db +F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e -R 1dffa8ce2c066e6a4b41c4abd6aac427 +P 17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 +R 254ab4db9b1e239ec16eb2e1a31dc657 U drh -Z 2a27a64568a31340c97915e2beb3adb7 +Z 489a805b1f3c740affbc27a0febf881c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7f494549c9..d0a052a41d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 \ No newline at end of file +b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6ab9230d83..589300c6ba 100644 --- a/src/where.c +++ b/src/where.c @@ -3094,7 +3094,7 @@ static int whereLoopAddBtreeIndex( assert( pSrc->pTab->szTabRow>0 ); rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); @@ -3270,9 +3270,12 @@ static int exprIsCoveredByIndex( /* ** Structure passed to the whereIsCoveringIndex Walker callback. */ +typedef struct CoveringIndexCheck CoveringIndexCheck; struct CoveringIndexCheck { Index *pIdx; /* The index */ int iTabCur; /* Cursor number for the corresponding table */ + u8 bExpr; /* Uses an indexed expression */ + u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ }; /* @@ -3293,24 +3296,28 @@ struct CoveringIndexCheck { ** matches pExpr, then prune the search. */ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ - int i; /* Loop counter */ - const Index *pIdx; /* The index of interest */ - const i16 *aiColumn; /* Columns contained in the index */ - u16 nColumn; /* Number of columns in the index */ - pIdx = pWalk->u.pCovIdxCk->pIdx; + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + CoveringIndexCheck *pCk; /* Info about this search */ + + pCk = pWalk->u.pCovIdxCk; + pIdx = pCk->pIdx; if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ - if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; pIdx = pWalk->u.pCovIdxCk->pIdx; aiColumn = pIdx->aiColumn; nColumn = pIdx->nColumn; for(i=0; iiColumn ) return WRC_Continue; } - pWalk->eCode = 1; + pCk->bUnidx = 1; return WRC_Abort; }else if( pIdx->bHasExpr && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + pCk->bExpr = 1; return WRC_Prune; } return WRC_Continue; @@ -3319,30 +3326,39 @@ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ /* ** pIdx is an index that covers all of the low-number columns used by -** pWInfo->pSelect (columns from 0 through 62). But there are columns -** in pWInfo->pSelect beyond 62. This routine tries to answer the question -** of whether pIdx covers *all* columns in the query. +** pWInfo->pSelect (columns from 0 through 62) or an index that has +** expressions terms. Hence, we cannot determine whether or not it is +** a covering index by using the colUsed bitmasks. We have to do a search +** to see if the index is covering. This routine does that search. ** -** Return 0 if pIdx is a covering index. Return non-zero if pIdx is -** not a covering index or if we are unable to determine if pIdx is a -** covering index. +** The return value is one of these: ** -** This routine is an optimization. It is always safe to return non-zero. -** But returning zero when non-zero should have been returned can lead to -** incorrect bytecode and assertion faults. +** 0 The index is definitely not a covering index +** +** WHERE_IDX_ONLY The index is definitely a covering index +** +** WHERE_EXPRIDX The index is likely a covering index, but it is +** difficult to determine precisely because of the +** expressions that are indexed. Score it as a +** covering index, but still keep the main table open +** just in case we need it. +** +** This routine is an optimization. It is always safe to return zero. +** But returning one of the other two values when zero should have been +** returned can lead to incorrect bytecode and assertion faults. */ static SQLITE_NOINLINE u32 whereIsCoveringIndex( WhereInfo *pWInfo, /* The WHERE clause context */ Index *pIdx, /* Index that is being tested */ int iTabCur /* Cursor for the table being indexed */ ){ - int i; + int i, rc; struct CoveringIndexCheck ck; Walker w; if( pWInfo->pSelect==0 ){ /* We don't have access to the full query, so we cannot check to see ** if pIdx is covering. Assume it is not. */ - return 1; + return 0; } if( pIdx->bHasExpr==0 ){ for(i=0; inColumn; i++){ @@ -3352,18 +3368,26 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( /* pIdx does not index any columns greater than 62, but we know from ** colMask that columns greater than 62 are used, so this is not a ** covering index */ - return 1; + return 0; } } ck.pIdx = pIdx; ck.iTabCur = iTabCur; + ck.bExpr = 0; + ck.bUnidx = 0; memset(&w, 0, sizeof(w)); w.xExprCallback = whereIsCoveringIndexWalkCallback; w.xSelectCallback = sqlite3SelectWalkNoop; w.u.pCovIdxCk = &ck; - w.eCode = 0; sqlite3WalkSelect(&w, pWInfo->pSelect); - return w.eCode; + if( ck.bUnidx ){ + rc = 0; + }else if( ck.bExpr ){ + rc = WHERE_EXPRIDX; + }else{ + rc = WHERE_IDX_ONLY; + } + return rc; } /* @@ -3579,21 +3603,38 @@ static int whereLoopAddBtree( }else{ Bitmask m; if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol) ){ - m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + pNew->wsFlags = WHERE_INDEXED; + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ + u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + if( isCov==0 ){ + WHERETRACE(0xff, + ("-> %s is not a covering index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + assert( m!=0 ); + }else{ + m = 0; + pNew->wsFlags |= isCov; + if( isCov & WHERE_IDX_ONLY ){ + WHERETRACE(0xff, + ("-> %s is a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + }else{ + assert( isCov==WHERE_EXPRIDX ); + WHERETRACE(0xff, + ("-> %s might be a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + } + } + }else if( m==0 ){ WHERETRACE(0xff, - ("-> %s %s a covering index according to whereIsCoveringIndex()\n", - pProbe->zName, m==0 ? "is" : "is not")); - }else{ - WHERETRACE(0xff, - ("-> %s %s a covering index according to bitmasks\n", + ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; } - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } /* Full scan via index */ diff --git a/src/whereInt.h b/src/whereInt.h index 28ede5be66..b89a4513e3 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -634,5 +634,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ #define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ +#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ From cbb958580575c4f4c57dc2019993d44653e04f41 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 18:17:42 +0000 Subject: [PATCH 55/72] Test cases added derived from the enhancment ticket. FossilOrigin-Name: c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 --- manifest | 12 +++---- manifest.uuid | 2 +- test/indexexpr1.test | 83 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index c83d4173b2..c76ad7a312 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\scovering-index-expr\sbranch. -D 2022-11-28T16:11:10.036 +C Test\scases\sadded\sderived\sfrom\sthe\senhancment\sticket. +D 2022-11-28T18:17:42.151 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1213,7 +1213,7 @@ F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6 F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 -F test/indexexpr1.test 3360c2a29a8844e7c4b13293567025281257f9e13a31854cfff6959cede11502 +F test/indexexpr1.test 1cd460113e3999a1477754fd0e3ac0d7bcbbbc742b427b418945f28af708c695 F test/indexexpr2.test 2c7abe3c48f8aaa5a448615ab4d13df3662185d28419c00999670834a3f0b484 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a -R 5944640ae5b4a3237e1d8a4c46c48db4 +P 89d775ada39aed4dc532374ace02156d07dc06e8ae54a194608af0c5a582d20f +R e73156e3b8e8d088eb2bbd902f358640 U drh -Z 9aa95e661b7edb77eed478b829e7796d +Z b96271db8dd85961a10ccefd57c768e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e7c719444e..a270d0f31b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89d775ada39aed4dc532374ace02156d07dc06e8ae54a194608af0c5a582d20f \ No newline at end of file +c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 \ No newline at end of file diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 042132b81d..a5d1514bcd 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -420,7 +420,7 @@ do_execsql_test indexexpr1-1500 { # 2018-01-03 OSSFuzz discovers another test case for the same problem # above. # -do_execsql_test indexexpr-1510 { +do_execsql_test indexexpr1-1510 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(a PRIMARY KEY,b UNIQUE); REPLACE INTO t1 VALUES(2, 1); @@ -434,18 +434,18 @@ do_execsql_test indexexpr-1510 { # a numeric table column, trouble can arise since there are multiple # string that can map to the same numeric value. (Ex: 123, 0123, 000123). # -do_execsql_test indexexpr-1600 { +do_execsql_test indexexpr1-1600 { DROP TABLE IF EXISTS t1; CREATE TABLE t1 (a INTEGER, b); CREATE INDEX idx1 ON t1 (lower(a)); INSERT INTO t1 VALUES('0001234',3); PRAGMA integrity_check; } {ok} -do_execsql_test indexexpr-1610 { +do_execsql_test indexexpr1-1610 { INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1); SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b; } {0 1 2 3} -do_execsql_test indexexpr-1620 { +do_execsql_test indexexpr1-1620 { SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b; } {} @@ -453,7 +453,7 @@ do_execsql_test indexexpr-1620 { # ExprImpliesExpr theorem prover bug: # "(NULL IS FALSE) IS FALSE" does not imply "NULL IS NULL" # -do_execsql_test indexexpr-1700 { +do_execsql_test indexexpr1-1700 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0); INSERT INTO t0(c0) VALUES (0); @@ -468,17 +468,17 @@ do_execsql_test indexexpr-1700 { # computing the expression. # ifcapable like_match_blobs { - do_execsql_test indexexpr-1800 { + do_execsql_test indexexpr1-1800 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0 REAL, c1 TEXT); CREATE INDEX i0 ON t0(+c0, c0); INSERT INTO t0(c0) VALUES(0); SELECT CAST(+ t0.c0 AS BLOB) LIKE 0 FROM t0; } {0} - do_execsql_test indexexpr-1810 { + do_execsql_test indexexpr1-1810 { SELECT CAST(+ t0.c0 AS BLOB) LIKE '0.0' FROM t0; } {1} - do_execsql_test indexexpr-1820 { + do_execsql_test indexexpr1-1820 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(x REAL); CREATE INDEX t1x ON t1(x, +x); @@ -491,19 +491,80 @@ ifcapable like_match_blobs { # Assertion fault during a DELETE INDEXED BY. # reset_db -do_execsql_test indexexpr-1900 { +do_execsql_test indexexpr1-1900 { CREATE TABLE t1(x TEXT PRIMARY KEY, y TEXT, z INT); INSERT INTO t1(x,y,z) VALUES('alpha','ALPHA',1),('bravo','charlie',1); CREATE INDEX i1 ON t1(+y COLLATE NOCASE); SELECT * FROM t1; } {alpha ALPHA 1 bravo charlie 1} -do_execsql_test indexexpr-1910 { +do_execsql_test indexexpr1-1910 { DELETE FROM t1 INDEXED BY i1 WHERE x IS +y COLLATE NOCASE IN (SELECT z FROM t1) RETURNING *; } {alpha ALPHA 1} -do_execsql_test indexexpr-1920 { +do_execsql_test indexexpr1-1920 { SELECT * FROM t1; } {bravo charlie 1} +# 2022-11-28 Ticket 695a1a53de +# Improved ability to recognize that an index on an expression is a +# covering index. +# +reset_db +do_execsql_test indexexpr1-2000 { + CREATE TABLE t1(a INT, b TEXT); + INSERT INTO t1(a,b) VALUES + (10, '{"one":5,"two":6}'), + (10, '{"one":50,"two":60}'), + (10, '{"three":99}'), + (11, '{"one":100,"two":200}'); + CREATE INDEX t1_one ON t1(a, b->>'one'); + CREATE INDEX t1_two ON t1(a, b->>'two'); +} +do_execsql_test indexexpr1-2010 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {/.* t1_one .*/} +do_execsql_test indexexpr1-2011 { + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {55} +do_execsql_test indexexpr1-2020 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {/.* t1_two .*/} +do_execsql_test indexexpr1-2021 { + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {66} +do_execsql_test indexexpr1-2030 { + DROP TABLE t1; + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} +do_execsql_test indexexpr1-2030 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {1 6 4 54 46} +do_execsql_test indexexpr1-2030 { + explain query plan + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {/.*SCAN t1 USING INDEX t1x.*/} + + + finish_test From 1b3d13e65e125500d1034e889b37363baa8f1e38 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Nov 2022 18:41:41 +0000 Subject: [PATCH 56/72] Add the SQLITE_FCNTL_RESET_CACHE verb. Use it to ensure that the page cache is purged before and after a the recovery extension is run. FossilOrigin-Name: 6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f --- ext/recover/recover1.test | 41 ++++++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.c | 6 +++++- manifest | 24 ++++++++++----------- manifest.uuid | 2 +- src/btree.c | 11 ++++++++++ src/btree.h | 2 ++ src/main.c | 3 +++ src/sqlite.h.in | 7 ++++++ 8 files changed, 82 insertions(+), 14 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 75f5dba1ff..3e8a691492 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -273,5 +273,46 @@ do_execsql_test 15.1 { } {} do_recover_test 15 +#------------------------------------------------------------------------- +reset_db +do_execsql_test 16.1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1), (2), (3); +} {wal} +do_test 16.2 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} +do_execsql_test 16.3 { + SELECT * FROM t1; +} {1 2 3} + +do_execsql_test 16.4 { + BEGIN; + SELECT * FROM t1; +} {1 2 3} +do_test 16.5 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {1 {cannot start a transaction within a transaction}} +do_execsql_test 16.6 { + SELECT * FROM t1; +} {1 2 3} +do_execsql_test 16.7 { + INSERT INTO t1 VALUES(4); +} +do_test 16.8 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {1 {cannot start a transaction within a transaction}} +do_execsql_test 16.9 { + SELECT * FROM t1; + COMMIT; +} {1 2 3 4} + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 30260f014e..e62aba752f 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -2017,6 +2017,7 @@ static void recoverFinalCleanup(sqlite3_recover *p){ p->pTblList = 0; sqlite3_finalize(p->pGetPage); p->pGetPage = 0; + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); { #ifndef NDEBUG @@ -2315,6 +2316,7 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ ** ** + first freelist page (32-bits at offset 32) ** + size of freelist (32-bits at offset 36) + ** + the wal-mode flags (16-bits at offset 18) ** ** We also try to preserve the auto-vacuum, incr-value, user-version ** and application-id fields - all 32 bit quantities at offsets @@ -2378,7 +2380,8 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ if( p->pPage1Cache ){ p->pPage1Disk = &p->pPage1Cache[nByte]; memcpy(p->pPage1Disk, aBuf, nByte); - + aHdr[18] = a[18]; + aHdr[19] = a[19]; recoverPutU32(&aHdr[28], dbsz); recoverPutU32(&aHdr[56], enc); recoverPutU16(&aHdr[105], pgsz-nReserve); @@ -2574,6 +2577,7 @@ static void recoverStep(sqlite3_recover *p){ recoverOpenOutput(p); /* Open transactions on both the input and output databases. */ + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); recoverExec(p, p->dbIn, "BEGIN"); if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; diff --git a/manifest b/manifest index 477c836f77..a159026919 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s(optional)\sbase64\sand\sbase85\sUDF\sextensions. -D 2022-11-28T14:51:50.439 +C Add\sthe\sSQLITE_FCNTL_RESET_CACHE\sverb.\sUse\sit\sto\sensure\sthat\sthe\spage\scache\sis\spurged\sbefore\sand\safter\sa\sthe\srecovery\sextension\sis\srun. +D 2022-11-28T18:41:41.586 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -390,7 +390,7 @@ F ext/rbu/sqlite3rbu.c c4ba7901b2d3e0c7845f30840e3ffb35c6f999d6da0d80f121866491f F ext/rbu/sqlite3rbu.h 02d981e2d39c151391759e1a400e29c7388730812957ac3db8dad7f6c9f9cfc8 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 F ext/recover/dbdata.c 8f1f75d636431de69d7977ec50fc41bfdd0c48c510d5ee7eae0cbd4164e1429a -F ext/recover/recover1.test 02004eb8f9ec2825ba77e24742c18e45162cb21d27e76a3a435b83a759a1131a +F ext/recover/recover1.test 2a2df2943d6696f9487e75868feae4b1511c4a511b102854ba0d2af0326d9dfb F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 @@ -402,7 +402,7 @@ F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1b F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c 1e4de9cd30daa67e5c1cd0cafb764ddf183087ce037ab47b7c1a7e2921c90d37 +F ext/recover/sqlite3recover.c 6d37a422c8917b793548a7b2c6e7ff8e1af15ba5453eef37bb69331828b73186 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 1a34e2d04533d919a30ae4d5caeb1643f6684e9ccd7597ca27721d8af81f4ade F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -583,8 +583,8 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 522df0f1173495e06c5589f0b17a61f53a212017b82be53171133fcfc0e1e90a -F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de +F src/btree.c d7d74e6c87a2d947bbf37c705a5ac11b0dc7b3848e76d0ff7c940bb644bba291 +F src/btree.h 49da925329574798be3cbb745a49d069a9e67c99900d8a0d04b1e934d60394ea F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a @@ -607,7 +607,7 @@ F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 25663175950c5c4404b9377840b7b4c6fe5c53b415caf43634c62f442c02a9a7 -F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe +F src/main.c 954490392b74fb215378af3c75a9e1f4f559f19cb1567e5d77f3fbbb63909b4d F src/malloc.c 3d4ec162214024ee071d85711b93bec25cd3371280aee3702b63bcf312ca8238 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -646,7 +646,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c -F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca +F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa b8345630a2a322234bda49ee4b996f6ba20e2b080621e229a2ec5e820892a663 -R b25d12f698cc565856f0ef09be44e05e -U larrybr -Z dca654d0507bf9deef15f6de6a6be007 +P b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a +R 013988a2db147f7833ada18759e80a5f +U dan +Z 0974da01907b06c2d097fc0b34ec52e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d5baba61d..030eff1202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a \ No newline at end of file +6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index cabcf675e3..c949001b89 100644 --- a/src/btree.c +++ b/src/btree.c @@ -11083,6 +11083,17 @@ int sqlite3BtreeIsReadonly(Btree *p){ */ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } +/* +** If no transaction is active and the database is not a temp-db, clear +** the in-memory pager cache. +*/ +void sqlite3BtreeClearCache(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->inTransaction==TRANS_NONE ){ + sqlite3PagerClearCache(pBt->pPager); + } +} + #if !defined(SQLITE_OMIT_SHARED_CACHE) /* ** Return true if the Btree passed as the only argument is sharable. diff --git a/src/btree.h b/src/btree.h index 7f31c6020f..c9b9d80175 100644 --- a/src/btree.h +++ b/src/btree.h @@ -368,6 +368,8 @@ void sqlite3BtreeCursorList(Btree*); int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); +void sqlite3BtreeClearCache(Btree*); + /* ** If we are not using shared cache, then there is no need to ** use mutexes to access the BtShared structures. So make the diff --git a/src/main.c b/src/main.c index b0645efe61..67dd60ae7e 100644 --- a/src/main.c +++ b/src/main.c @@ -3950,6 +3950,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); } rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESET_CACHE ){ + sqlite3BtreeClearCache(pBtree); + rc = SQLITE_OK; }else{ int nSave = db->busyHandler.nBusy; rc = sqlite3OsFileControl(fd, op, pArg); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c2fc4e5a6a..369d6b6022 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1192,6 +1192,12 @@ struct sqlite3_io_methods { ** **

  • [[SQLITE_FCNTL_CKSM_FILE]] ** Used by the cksmvfs VFS module only. +** +**
  • [[SQLITE_FCNTL_RESET_CACHE]] +** If there is currently no transaction open on the database, and the +** database is not a temp db, then this file-control purges the contents +** of the in-memory page cache. If there is an open transaction, or if +** the db is a temp-db, it is a no-op, not an error. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1234,6 +1240,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CKPT_START 39 #define SQLITE_FCNTL_EXTERNAL_READER 40 #define SQLITE_FCNTL_CKSM_FILE 41 +#define SQLITE_FCNTL_RESET_CACHE 42 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE From 5b046dadfcececc6416535de6a4e2a2dd25e3201 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 20:08:15 +0000 Subject: [PATCH 57/72] Fix harmless compiler warnings. FossilOrigin-Name: 8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 +++--- src/select.c | 9 +++++---- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d8cdd7a1e9..9e7f070df8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implementation\sof\senhancement\srequest\s[695a1a53dea6b240]:\sThe\squery\splanner\nhas\simproved\sawareness\sof\swhen\sindexes\son\sexpressions\sare\scovering\sand\sadjusts\ntheir\scosts\saccordingly. -D 2022-11-28T19:42:48.433 +C Fix\sharmless\scompiler\swarnings. +D 2022-11-28T20:08:15.505 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -583,7 +583,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c d7d74e6c87a2d947bbf37c705a5ac11b0dc7b3848e76d0ff7c940bb644bba291 +F src/btree.c bd4375adcc5e45b3267549ba201a04b2f41fbb0a581d33513fb186a208d4c81d F src/btree.h 49da925329574798be3cbb745a49d069a9e67c99900d8a0d04b1e934d60394ea F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -644,7 +644,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 +F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 -R cf4309a5723ca7c0ed03679edb5024d5 +P e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63 +R a17c4713fb6c6661faa720f3ee28899c U drh -Z 287040c8f243d0f300388e5a2dad7f24 +Z eb9319d9dfe7dee1c2083bff54e6e44f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e27e9da559..3479de78d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63 \ No newline at end of file +8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c949001b89..e71531c730 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7127,12 +7127,12 @@ static int insertCell( assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ - int rc = SQLITE_OK; + int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pPage, pCell, &rc); - if( rc ) return rc; + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc; } #endif } diff --git a/src/select.c b/src/select.c index 10498310d4..0cd28b543e 100644 --- a/src/select.c +++ b/src/select.c @@ -6290,7 +6290,6 @@ static void printAggInfo(AggInfo *pAggInfo){ ** (or recomputing) those aCol[] entries. */ static void analyzeAggFuncArgs( - Parse *pParse, AggInfo *pAggInfo, NameContext *pNC ){ @@ -6335,7 +6334,7 @@ static void optimizeAggregateUseOfIndexedExpr( pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; } } - analyzeAggFuncArgs(pParse, pAggInfo, pNC); + analyzeAggFuncArgs(pAggInfo, pNC); #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ IndexedExpr *pIEpr; @@ -6350,7 +6349,8 @@ static void optimizeAggregateUseOfIndexedExpr( printAggInfo(pAggInfo); } #else - (void)pSelect; /* Suppress unused-parameter warnings */ + UNUSED_PARAMETER(pSelect); + UNUSED_PARAMETER(pParse); #endif } @@ -6360,6 +6360,7 @@ static void optimizeAggregateUseOfIndexedExpr( static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo; struct AggInfo_col *pCol; + UNUSED_PARAMETER(pWalker); if( pExpr->pAggInfo==0 ) return WRC_Continue; if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; @@ -7611,7 +7612,7 @@ int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - analyzeAggFuncArgs(pParse, pAggInfo, &sNC); + analyzeAggFuncArgs(pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ From 0b0070f1bdf78f7c9ff8315f1aa635592205f9bd Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 28 Nov 2022 21:09:49 +0000 Subject: [PATCH 58/72] Get ext/misc/basexx.c into the testfixture.exe build for MSC. FossilOrigin-Name: bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c --- Makefile.msc | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 191701cdba..0312b0dbaf 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1561,6 +1561,7 @@ TESTEXT = \ $(TOP)\ext\expert\test_expert.c \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\appendvfs.c \ + $(TOP)\ext\misc\basexx.c \ $(TOP)\ext\misc\carray.c \ $(TOP)\ext\misc\cksumvfs.c \ $(TOP)\ext\misc\closure.c \ diff --git a/manifest b/manifest index 9e7f070df8..895ea017fd 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Fix\sharmless\scompiler\swarnings. -D 2022-11-28T20:08:15.505 +C Get\sext/misc/basexx.c\sinto\sthe\stestfixture.exe\sbuild\sfor\sMSC. +D 2022-11-28T21:09:49.816 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in c223963d7b0828f26cb62ea3e0f583d26839b7d3ef0d1cca87f35c4b222ff01b F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc e7a564ceec71f0d9666031d5638cf0d4f88b050b44e8df5d32125137cd259ac0 +F Makefile.msc e86ec21328921721dd6f777da435c62a5469cda1a654f5ecefacf87ddb3dfeb3 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 413ec94920a487ae32c9a2a8819544d690662d6f7c7ce025c0d0b8a1e74fa9db F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63 -R a17c4713fb6c6661faa720f3ee28899c -U drh -Z eb9319d9dfe7dee1c2083bff54e6e44f +P 8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 +R 6f236bfe43856f6e65bad104d646e92b +U larrybr +Z a8a43d921cd6c44f25932dfa5b1108ac # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3479de78d9..46da65e1fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 \ No newline at end of file +bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c \ No newline at end of file From d3fc2e63aa45f0a5a64baac15242aa369a74cfb3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 21:17:30 +0000 Subject: [PATCH 59/72] Check-in [8d5b76593d82b3a5] contained an error that was causing some obscure error codes to be lost. Fixed here. FossilOrigin-Name: 46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 895ea017fd..04e653dcec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sext/misc/basexx.c\sinto\sthe\stestfixture.exe\sbuild\sfor\sMSC. -D 2022-11-28T21:09:49.816 +C Check-in\s[8d5b76593d82b3a5]\scontained\san\serror\sthat\swas\scausing\ssome\sobscure\nerror\scodes\sto\sbe\slost.\s\sFixed\shere. +D 2022-11-28T21:17:30.250 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -583,7 +583,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c bd4375adcc5e45b3267549ba201a04b2f41fbb0a581d33513fb186a208d4c81d +F src/btree.c 2f794c217e52fdf4322bf37ee7778331b4d93aed2c00b5d67f914c0239a9edcc F src/btree.h 49da925329574798be3cbb745a49d069a9e67c99900d8a0d04b1e934d60394ea F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 -R 6f236bfe43856f6e65bad104d646e92b -U larrybr -Z a8a43d921cd6c44f25932dfa5b1108ac +P bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c +R edc56f3db34be700dc2fe2c48f4c3e31 +U drh +Z fbd7482d8e22e645fc260e0f000cee41 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 46da65e1fd..2763f271ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c \ No newline at end of file +46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e71531c730..d38b7a551a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7132,7 +7132,7 @@ static int insertCell( ** the entry for the overflow page into the pointer map. */ ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); - if( rc2 ) return rc; + if( rc2 ) return rc2; } #endif } From e6b0154138801768597d572c298af899ffebddeb Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 02:23:12 +0000 Subject: [PATCH 60/72] Add an explicit warning about the current API-instability of the sqlite3.opfs namespace, which may need to be eliminated based on re-thinking of how the OPFS sqlite3_vfs is registered. Comment changes only - no code. FossilOrigin-Name: 0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 --- ext/wasm/api/sqlite3-api-opfs.js | 7 +++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 5537df89fb..458448175b 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -128,6 +128,10 @@ const installOpfsVfs = function callee(options){ /** Generic utilities for working with OPFS. This will get filled out by the Promise setup and, on success, installed as sqlite3.opfs. + + ACHTUNG: do not rely on these APIs in client code. They are + experimental and subject to change or removal as the + OPFS-specific sqlite3_vfs evolves. */ const opfsUtil = Object.create(null); /** @@ -186,8 +190,7 @@ const installOpfsVfs = function callee(options){ const dVfs = pDVfs ? new sqlite3_vfs(pDVfs) : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. Though we cannot currently handle - that case, the hope is to eventually be able to. */; + SQLITE_OS_OTHER. */; const opfsVfs = new sqlite3_vfs(); const opfsIoMethods = new sqlite3_io_methods(); opfsVfs.$iVersion = 2/*yes, two*/; diff --git a/manifest b/manifest index 04e653dcec..01aaf2a316 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check-in\s[8d5b76593d82b3a5]\scontained\san\serror\sthat\swas\scausing\ssome\sobscure\nerror\scodes\sto\sbe\slost.\s\sFixed\shere. -D 2022-11-28T21:17:30.250 +C Add\san\sexplicit\swarning\sabout\sthe\scurrent\sAPI-instability\sof\sthe\ssqlite3.opfs\snamespace,\swhich\smay\sneed\sto\sbe\seliminated\sbased\son\sre-thinking\sof\show\sthe\sOPFS\ssqlite3_vfs\sis\sregistered.\sComment\schanges\sonly\s-\sno\scode. +D 2022-11-29T02:23:12.943 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7 +F ext/wasm/api/sqlite3-api-opfs.js 3cdae7e98c500f89f9468a260e2a0e1b528c845a107bf72d368e5222769214d3 F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c -R edc56f3db34be700dc2fe2c48f4c3e31 -U drh -Z fbd7482d8e22e645fc260e0f000cee41 +P 46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 +R 3870d04bfd54da096e986662fe29b1c8 +U stephan +Z ad4a8c5f45a34a10f588c0c6dc455846 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2763f271ba..361de4178e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 \ No newline at end of file +0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 \ No newline at end of file From 04184761de820ac763036157ff07b2f22a89db77 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 05:25:08 +0000 Subject: [PATCH 61/72] Internal restructuring of the OPFS sqlite3_vfs in order to facilitate certain experimentation and improve error reporting/hints if it cannot be activated. Deprecate the name sqlite3.opfs.OpfsDb, preferring sqlite3.oo1.OpfsDb for consistency with JsStorageDb and any future DB subclasses. FossilOrigin-Name: 0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 --- ext/wasm/api/sqlite3-api-opfs.js | 79 +- ext/wasm/api/sqlite3-opfs-async-proxy.js | 1655 ++++++++++---------- ext/wasm/tester1.c-pp.js | 34 +- ext/wasm/tests/opfs/concurrency/index.html | 7 +- manifest | 18 +- manifest.uuid | 2 +- 6 files changed, 919 insertions(+), 876 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 458448175b..cef15305d4 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -76,15 +76,25 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `opfs` property, containing several OPFS-specific utilities. */ const installOpfsVfs = function callee(options){ - if(!self.SharedArrayBuffer || - !self.Atomics || - !self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ + if(!self.SharedArrayBuffer + || !self.Atomics){ return Promise.reject( - new Error("This environment does not have OPFS support.") + new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ + "The server must emit the COOP/COEP response headers to enable those. "+ + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") + ); + }else if(self.window===self && self.document){ + return Promise.reject( + new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ + "because it requires Atomics.wait().") + ); + }else if(!self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + return Promise.reject( + new Error("Missing required OPFS APIs.") ); } if(!options || 'object'!==typeof options){ @@ -134,6 +144,18 @@ const installOpfsVfs = function callee(options){ OPFS-specific sqlite3_vfs evolves. */ const opfsUtil = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + const thisThreadHasOPFS = ()=>{ + return self.FileSystemHandle && + self.FileSystemDirectoryHandle && + self.FileSystemFileHandle && + self.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator.storage.getDirectory; + }; + /** Not part of the public API. Solely for internal/development use. @@ -1179,12 +1201,16 @@ const installOpfsVfs = function callee(options){ //consideration. if(sqlite3.oo1){ - opfsUtil.OpfsDb = function(...args){ + const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); opt.vfs = opfsVfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; - opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsDb = + opfsUtil.OpfsDb /* sqlite3.opfs.OpfsDb => deprecated name - + will be phased out Real Soon */ = + OpfsDb; + OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, [ @@ -1206,13 +1232,6 @@ const installOpfsVfs = function callee(options){ ); } - /** - Potential TODOs: - - - Expose one or both of the Worker objects via opfsUtil and - publish an interface for proxying the higher-level OPFS - features like getting a directory listing. - */ const sanityCheck = function(){ const scope = wasm.scopedAllocPush(); const sq3File = new sqlite3_file(); @@ -1282,6 +1301,11 @@ const installOpfsVfs = function callee(options){ W.onmessage = function({data}){ //log("Worker.onmessage:",data); switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; case 'opfs-async-loaded': /*Arrives as soon as the asyc proxy finishes loading. Pass our config and shared state on to the async worker.*/ @@ -1308,14 +1332,18 @@ const installOpfsVfs = function callee(options){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); } - navigator.storage.getDirectory().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); + if(thisThreadHasOPFS()){ + navigator.storage.getDirectory().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + sqlite3.opfs = opfsUtil; + opfsUtil.rootDirectory = d; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(sqlite3); + }); + }else{ promiseResolve(sqlite3); - }); + } }catch(e){ error(e); promiseReject(e); @@ -1334,9 +1362,6 @@ const installOpfsVfs = function callee(options){ installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - if(sqlite3.scriptInfo && !sqlite3.scriptInfo.isWorker){ - return; - } try{ let proxyJs = installOpfsVfs.defaultProxyUri; if(sqlite3.scriptInfo.sqlite3Dir){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8e60969bc6..1ba6e9bdbb 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -47,854 +47,871 @@ usage of those methods to remove the "await". */ "use strict"; -const toss = function(...args){throw new Error(args.join(' '))}; -if(self.window === self){ - toss("This code cannot run from the main thread.", - "Load it as a Worker from a separate Worker."); -}else if(!navigator.storage.getDirectory){ - toss("This API requires navigator.storage.getDirectory."); -} - -/** - Will hold state copied to this object from the syncronous side of - this API. -*/ -const state = Object.create(null); - -/** - verbose: - - 0 = no logging output - 1 = only errors - 2 = warnings and errors - 3 = debug, warnings, and errors -*/ -state.verbose = 1; - -const loggers = { - 0:console.error.bind(console), - 1:console.warn.bind(console), - 2:console.log.bind(console) -}; -const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); -}; -const log = (...args)=>logImpl(2, ...args); -const warn = (...args)=>logImpl(1, ...args); -const error = (...args)=>logImpl(0, ...args); -const metrics = Object.create(null); -metrics.reset = ()=>{ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); +const wPost = (type,...args)=>postMessage({type, payload:args}); +const installAsyncProxy = function(self){ + const toss = function(...args){throw new Error(args.join(' '))}; + if(self.window === self){ + toss("This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); + }else if(!navigator.storage.getDirectory){ + toss("This API requires navigator.storage.getDirectory."); } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; -}; -metrics.dump = ()=>{ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - } - console.log(self.location.href, - "metrics for",self.location.href,":\n", - metrics, - "\nTotal of",n,"op(s) for",t,"ms", - "approx",w,"ms spent waiting on OPFS APIs."); - console.log("Serialization metrics:",metrics.s11n); -}; -/** - __openFiles is a map of sqlite3_file pointers (integers) to - metadata related to a given OPFS file handles. The pointers are, in - this side of the interface, opaque file handle IDs provided by the - synchronous part of this constellation. Each value is an object - with a structure demonstrated in the xOpen() impl. -*/ -const __openFiles = Object.create(null); -/** - __implicitLocks is a Set of sqlite3_file pointers (integers) which were - "auto-locked". i.e. those for which we obtained a sync access - handle without an explicit xLock() call. Such locks will be - released during db connection idle time, whereas a sync access - handle obtained via xLock(), or subsequently xLock()'d after - auto-acquisition, will not be released until xUnlock() is called. - - Maintenance reminder: if we relinquish auto-locks at the end of the - operation which acquires them, we pay a massive performance - penalty: speedtest1 benchmarks take up to 4x as long. By delaying - the lock release until idle time, the hit is negligible. -*/ -const __implicitLocks = new Set(); - -/** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg is - true, the result is returned as an array of path elements, else an - absolute path string is returned. -*/ -const getResolvedPath = function(filename,splitIt){ - const p = new URL( - filename, 'file://irrelevant' - ).pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; -}; - -/** - Takes the absolute path to a filesystem element. Returns an array - of [handleOfContainingDir, filename]. If the 2nd argument is truthy - then each directory element leading to the file is created along - the way. Throws if any creation or resolution fails. -*/ -const getDirForFilename = async function f(absFilename, createDirs = false){ - const path = getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = state.rootDir; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; -}; - -/** - If the given file-holding object has a sync handle attached to it, - that handle is remove and asynchronously closed. Though it may - sound sensible to continue work as soon as the close() returns - (noting that it's asynchronous), doing so can cause operations - performed soon afterwards, e.g. a call to getSyncHandle() to fail - because they may happen out of order from the close(). OPFS does - not guaranty that the actual order of operations is retained in - such cases. i.e. always "await" on the result of this function. -*/ -const closeSyncHandle = async (fh)=>{ - if(fh.syncHandle){ - log("Closing sync handle for",fh.filenameAbs); - const h = fh.syncHandle; - delete fh.syncHandle; - delete fh.xLock; - __implicitLocks.delete(fh.fid); - return h.close(); - } -}; - -/** - A proxy for closeSyncHandle() which is guaranteed to not throw. - - This function is part of a lock/unlock step in functions which - require a sync access handle but may be called without xLock() - having been called first. Such calls need to release that - handle to avoid locking the file for all of time. This is an - _attempt_ at reducing cross-tab contention but it may prove - to be more of a problem than a solution and may need to be - removed. -*/ -const closeSyncHandleNoThrow = async (fh)=>{ - try{await closeSyncHandle(fh)} - catch(e){ - warn("closeSyncHandleNoThrow() ignoring:",e,fh); - } -}; - -/* Release all auto-locks. */ -const releaseImplicitLocks = async ()=>{ - if(__implicitLocks.size){ - /* Release all auto-locks. */ - for(const fid of __implicitLocks){ - const fh = __openFiles[fid]; - await closeSyncHandleNoThrow(fh); - log("Auto-unlocked",fid,fh.filenameAbs); - } - } -}; - -/** - An experiment in improving concurrency by freeing up implicit locks - sooner. This is known to impact performance dramatically but it has - also shown to improve concurrency considerably. - - If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, - this routine returns closeSyncHandleNoThrow(), else it is a no-op. -*/ -const releaseImplicitLock = async (fh)=>{ - if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ - return closeSyncHandleNoThrow(fh); - } -}; - -/** - An error class specifically for use with getSyncHandle(), the goal - of which is to eventually be able to distinguish unambiguously - between locking-related failures and other types, noting that we - cannot currently do so because createSyncAccessHandle() does not - define its exceptions in the required level of detail. -*/ -class GetSyncHandleError extends Error { - constructor(errorObject, ...msg){ - super(); - this.error = errorObject; - this.message = [ - ...msg, ': Original exception ['+errorObject.name+']:', - errorObject.message - ].join(' '); - this.name = 'GetSyncHandleError'; - } -}; -GetSyncHandleError.convertRc = (e,rc)=>{ - if(0){ - /* This approach makes the very wild assumption that such a - failure _is_ a locking error. In practice that appears to be - the most common error, by far, but we cannot unambiguously - distinguish that from other errors. - - This approach is highly questionable. - */ - return (e instanceof GetSyncHandleError) - ? state.sq3Codes.SQLITE_IOERR_LOCK - : rc; - }else{ - return rc; - } -} -/** - Returns the sync access handle associated with the given file - handle object (which must be a valid handle object, as created by - xOpen()), lazily opening it if needed. - - In order to help alleviate cross-tab contention for a dabase, - if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to 3 times. If acquisition - still fails at that point it will give up and propagate the - exception. -*/ -const getSyncHandle = async (fh,opName)=>{ - if(!fh.syncHandle){ - const t = performance.now(); - log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, msBase = 300; - let i = 1, ms = msBase; - for(; true; ms = msBase * ++i){ - try { - //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); - //TODO? A config option which tells it to throw here - //randomly every now and then, for testing purposes. - fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); - break; - }catch(e){ - if(i === maxTries){ - throw new GetSyncHandleError( - e, "Error getting sync handle for",opName+"().",maxTries, - "attempts failed.",fh.filenameAbs - ); - } - warn("Error getting sync handle for",opName+"(). Waiting",ms, - "ms and trying again.",fh.filenameAbs,e); - //await releaseImplicitLocks(); - Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); - } - } - log("Got",opName+"() sync handle for",fh.filenameAbs, - 'in',performance.now() - t,'ms'); - if(!fh.xLock){ - __implicitLocks.add(fh.fid); - log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); - } - } - return fh.syncHandle; -}; - -/** - Stores the given value at state.sabOPView[state.opIds.rc] and then - Atomics.notify()'s it. -*/ -const storeAndNotify = (opName, value)=>{ - log(opName+"() => notify(",value,")"); - Atomics.store(state.sabOPView, state.opIds.rc, value); - Atomics.notify(state.sabOPView, state.opIds.rc); -}; - -/** - Throws if fh is a file-holding object which is flagged as read-only. -*/ -const affirmNotRO = function(opName,fh){ - if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); -}; -const affirmLocked = function(opName,fh){ - //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); /** - Currently a no-op, as speedtest1 triggers xRead() without a - lock (that seems like a bug but it's currently uninvestigated). - This means, however, that some OPFS VFS routines may trigger - acquisition of a lock but never let it go until xUnlock() is - called (which it likely won't be if xLock() was not called). + Will hold state copied to this object from the syncronous side of + this API. */ -}; + const state = Object.create(null); -/** - We track 2 different timers: the "metrics" timer records how much - time we spend performing work. The "wait" timer records how much - time we spend waiting on the underlying OPFS timer. See the calls - to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() - throughout this file to see how they're used. -*/ -const __mTimer = Object.create(null); -__mTimer.op = undefined; -__mTimer.start = undefined; -const mTimeStart = (op)=>{ - __mTimer.start = performance.now(); - __mTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); - ++metrics[op].count; -}; -const mTimeEnd = ()=>( - metrics[__mTimer.op].time += performance.now() - __mTimer.start -); -const __wTimer = Object.create(null); -__wTimer.op = undefined; -__wTimer.start = undefined; -const wTimeStart = (op)=>{ - __wTimer.start = performance.now(); - __wTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); -}; -const wTimeEnd = ()=>( - metrics[__wTimer.op].wait += performance.now() - __wTimer.start -); + /** + verbose: -/** - Gets set to true by the 'opfs-async-shutdown' command to quit the - wait loop. This is only intended for debugging purposes: we cannot - inspect this file's state while the tight waitLoop() is running and - need a way to stop that loop for introspection purposes. -*/ -let flagAsyncShutdown = false; + 0 = no logging output + 1 = only errors + 2 = warnings and errors + 3 = debug, warnings, and errors + */ + state.verbose = 1; - -/** - Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods - methods, as well as helpers like mkdir(). Maintenance reminder: - members are in alphabetical order to simplify finding them. -*/ -const vfsAsyncImpls = { - 'opfs-async-metrics': async ()=>{ - mTimeStart('opfs-async-metrics'); - metrics.dump(); - storeAndNotify('opfs-async-metrics', 0); - mTimeEnd(); - }, - 'opfs-async-shutdown': async ()=>{ - flagAsyncShutdown = true; - storeAndNotify('opfs-async-shutdown', 0); - }, - mkdir: async (dirname)=>{ - mTimeStart('mkdir'); - let rc = 0; - wTimeStart('mkdir'); - try { - await getDirForFilename(dirname+"/filepart", true); - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); + const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) + }; + const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const metrics = Object.create(null); + metrics.reset = ()=>{ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); } - storeAndNotify('mkdir', rc); - mTimeEnd(); - }, - xAccess: async (filename)=>{ - mTimeStart('xAccess'); - /* OPFS cannot support the full range of xAccess() queries sqlite3 - calls for. We can essentially just tell if the file is - accessible, but if it is it's automatically writable (unless - it's locked, which we cannot(?) know without trying to open - it). OPFS does not have the notion of read-only. - - The return semantics of this function differ from sqlite3's - xAccess semantics because we are limited in what we can - communicate back to our synchronous communication partner: 0 = - accessible, non-0 means not accessible. - */ - let rc = 0; - wTimeStart('xAccess'); - try{ - const [dh, fn] = await getDirForFilename(filename); - await dh.getFileHandle(fn); - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + }; + metrics.dump = ()=>{ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; } - storeAndNotify('xAccess', rc); - mTimeEnd(); - }, - xClose: async function(fid/*sqlite3_file pointer*/){ - const opName = 'xClose'; - mTimeStart(opName); - __implicitLocks.delete(fid); - const fh = __openFiles[fid]; - let rc = 0; - wTimeStart(opName); - if(fh){ - delete __openFiles[fid]; - await closeSyncHandle(fh); - if(fh.deleteOnClose){ - try{ await fh.dirHandle.removeEntry(fh.filenamePart) } - catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + console.log(self.location.href, + "metrics for",self.location.href,":\n", + metrics, + "\nTotal of",n,"op(s) for",t,"ms", + "approx",w,"ms spent waiting on OPFS APIs."); + console.log("Serialization metrics:",metrics.s11n); + }; + + /** + __openFiles is a map of sqlite3_file pointers (integers) to + metadata related to a given OPFS file handles. The pointers are, in + this side of the interface, opaque file handle IDs provided by the + synchronous part of this constellation. Each value is an object + with a structure demonstrated in the xOpen() impl. + */ + const __openFiles = Object.create(null); + /** + __implicitLocks is a Set of sqlite3_file pointers (integers) which were + "auto-locked". i.e. those for which we obtained a sync access + handle without an explicit xLock() call. Such locks will be + released during db connection idle time, whereas a sync access + handle obtained via xLock(), or subsequently xLock()'d after + auto-acquisition, will not be released until xUnlock() is called. + + Maintenance reminder: if we relinquish auto-locks at the end of the + operation which acquires them, we pay a massive performance + penalty: speedtest1 benchmarks take up to 4x as long. By delaying + the lock release until idle time, the hit is negligible. + */ + const __implicitLocks = new Set(); + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg is + true, the result is returned as an array of path elements, else an + absolute path string is returned. + */ + const getResolvedPath = function(filename,splitIt){ + const p = new URL( + filename, 'file://irrelevant' + ).pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an array + of [handleOfContainingDir, filename]. If the 2nd argument is truthy + then each directory element leading to the file is created along + the way. Throws if any creation or resolution fails. + */ + const getDirForFilename = async function f(absFilename, createDirs = false){ + const path = getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = state.rootDir; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); } + } + return [dh, filename]; + }; + + /** + If the given file-holding object has a sync handle attached to it, + that handle is remove and asynchronously closed. Though it may + sound sensible to continue work as soon as the close() returns + (noting that it's asynchronous), doing so can cause operations + performed soon afterwards, e.g. a call to getSyncHandle() to fail + because they may happen out of order from the close(). OPFS does + not guaranty that the actual order of operations is retained in + such cases. i.e. always "await" on the result of this function. + */ + const closeSyncHandle = async (fh)=>{ + if(fh.syncHandle){ + log("Closing sync handle for",fh.filenameAbs); + const h = fh.syncHandle; + delete fh.syncHandle; + delete fh.xLock; + __implicitLocks.delete(fh.fid); + return h.close(); + } + }; + + /** + A proxy for closeSyncHandle() which is guaranteed to not throw. + + This function is part of a lock/unlock step in functions which + require a sync access handle but may be called without xLock() + having been called first. Such calls need to release that + handle to avoid locking the file for all of time. This is an + _attempt_ at reducing cross-tab contention but it may prove + to be more of a problem than a solution and may need to be + removed. + */ + const closeSyncHandleNoThrow = async (fh)=>{ + try{await closeSyncHandle(fh)} + catch(e){ + warn("closeSyncHandleNoThrow() ignoring:",e,fh); + } + }; + + /* Release all auto-locks. */ + const releaseImplicitLocks = async ()=>{ + if(__implicitLocks.size){ + /* Release all auto-locks. */ + for(const fid of __implicitLocks){ + const fh = __openFiles[fid]; + await closeSyncHandleNoThrow(fh); + log("Auto-unlocked",fid,fh.filenameAbs); + } + } + }; + + /** + An experiment in improving concurrency by freeing up implicit locks + sooner. This is known to impact performance dramatically but it has + also shown to improve concurrency considerably. + + If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, + this routine returns closeSyncHandleNoThrow(), else it is a no-op. + */ + const releaseImplicitLock = async (fh)=>{ + if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ + return closeSyncHandleNoThrow(fh); + } + }; + + /** + An error class specifically for use with getSyncHandle(), the goal + of which is to eventually be able to distinguish unambiguously + between locking-related failures and other types, noting that we + cannot currently do so because createSyncAccessHandle() does not + define its exceptions in the required level of detail. + */ + class GetSyncHandleError extends Error { + constructor(errorObject, ...msg){ + super(); + this.error = errorObject; + this.message = [ + ...msg, ': Original exception ['+errorObject.name+']:', + errorObject.message + ].join(' '); + this.name = 'GetSyncHandleError'; + } + }; + GetSyncHandleError.convertRc = (e,rc)=>{ + if(0){ + /* This approach makes the very wild assumption that such a + failure _is_ a locking error. In practice that appears to be + the most common error, by far, but we cannot unambiguously + distinguish that from other errors. + + This approach is highly questionable. + */ + return (e instanceof GetSyncHandleError) + ? state.sq3Codes.SQLITE_IOERR_LOCK + : rc; }else{ - state.s11n.serialize(); - rc = state.sq3Codes.SQLITE_NOTFOUND; + return rc; } - wTimeEnd(); - storeAndNotify(opName, rc); - mTimeEnd(); - }, - xDelete: async function(...args){ - mTimeStart('xDelete'); - const rc = await vfsAsyncImpls.xDeleteNoWait(...args); - storeAndNotify('xDelete', rc); - mTimeEnd(); - }, - xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ - /* The syncDir flag is, for purposes of the VFS API's semantics, - ignored here. However, if it has the value 0x1234 then: after - deleting the given file, recursively try to delete any empty - directories left behind in its wake (ignoring any errors and - stopping at the first failure). + } + /** + Returns the sync access handle associated with the given file + handle object (which must be a valid handle object, as created by + xOpen()), lazily opening it if needed. - That said: we don't know for sure that removeEntry() fails if - the dir is not empty because the API is not documented. It has, - however, a "recursive" flag which defaults to false, so - presumably it will fail if the dir is not empty and that flag - is false. + In order to help alleviate cross-tab contention for a dabase, + if an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to 3 times. If acquisition + still fails at that point it will give up and propagate the + exception. + */ + const getSyncHandle = async (fh,opName)=>{ + if(!fh.syncHandle){ + const t = performance.now(); + log("Acquiring sync handle for",fh.filenameAbs); + const maxTries = 6, msBase = 300; + let i = 1, ms = msBase; + for(; true; ms = msBase * ++i){ + try { + //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); + //TODO? A config option which tells it to throw here + //randomly every now and then, for testing purposes. + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + break; + }catch(e){ + if(i === maxTries){ + throw new GetSyncHandleError( + e, "Error getting sync handle for",opName+"().",maxTries, + "attempts failed.",fh.filenameAbs + ); + } + warn("Error getting sync handle for",opName+"(). Waiting",ms, + "ms and trying again.",fh.filenameAbs,e); + //await releaseImplicitLocks(); + Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); + } + } + log("Got",opName+"() sync handle for",fh.filenameAbs, + 'in',performance.now() - t,'ms'); + if(!fh.xLock){ + __implicitLocks.add(fh.fid); + log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); + } + } + return fh.syncHandle; + }; + + /** + Stores the given value at state.sabOPView[state.opIds.rc] and then + Atomics.notify()'s it. + */ + const storeAndNotify = (opName, value)=>{ + log(opName+"() => notify(",value,")"); + Atomics.store(state.sabOPView, state.opIds.rc, value); + Atomics.notify(state.sabOPView, state.opIds.rc); + }; + + /** + Throws if fh is a file-holding object which is flagged as read-only. + */ + const affirmNotRO = function(opName,fh){ + if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); + }; + const affirmLocked = function(opName,fh){ + //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); + /** + Currently a no-op, as speedtest1 triggers xRead() without a + lock (that seems like a bug but it's currently uninvestigated). + This means, however, that some OPFS VFS routines may trigger + acquisition of a lock but never let it go until xUnlock() is + called (which it likely won't be if xLock() was not called). */ - let rc = 0; - wTimeStart('xDelete'); - try { - while(filename){ - const [hDir, filenamePart] = await getDirForFilename(filename, false); - if(!filenamePart) break; - await hDir.removeEntry(filenamePart, {recursive}); - if(0x1234 !== syncDir) break; - recursive = false; - filename = getResolvedPath(filename, true); - filename.pop(); - filename = filename.join('/'); - } - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_DELETE; - } - wTimeEnd(); - return rc; - }, - xFileSize: async function(fid/*sqlite3_file pointer*/){ - mTimeStart('xFileSize'); - const fh = __openFiles[fid]; - let rc; - wTimeStart('xFileSize'); - try{ - affirmLocked('xFileSize',fh); - const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); - state.s11n.serialize(Number(sz)); - rc = 0; - }catch(e){ - state.s11n.storeException(2,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); - } - await releaseImplicitLock(fh); - wTimeEnd(); - storeAndNotify('xFileSize', rc); - mTimeEnd(); - }, - xLock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xLock'); - const fh = __openFiles[fid]; - let rc = 0; - const oldLockType = fh.xLock; - fh.xLock = lockType; - if( !fh.syncHandle ){ - wTimeStart('xLock'); - try { - await getSyncHandle(fh,'xLock'); - __implicitLocks.delete(fid); - }catch(e){ - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); - fh.xLock = oldLockType; - } - wTimeEnd(); - } - storeAndNotify('xLock',rc); - mTimeEnd(); - }, - xOpen: async function(fid/*sqlite3_file pointer*/, filename, - flags/*SQLITE_OPEN_...*/, - opfsFlags/*OPFS_...*/){ - const opName = 'xOpen'; - mTimeStart(opName); - const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); - wTimeStart('xOpen'); - try{ - let hDir, filenamePart; - try { - [hDir, filenamePart] = await getDirForFilename(filename, !!create); - }catch(e){ - state.s11n.storeException(1,e); - storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); - mTimeEnd(); - wTimeEnd(); - return; - } - const hFile = await hDir.getFileHandle(filenamePart, {create}); - wTimeEnd(); - const fh = Object.assign(Object.create(null),{ - fid: fid, - filenameAbs: filename, - filenamePart: filenamePart, - dirHandle: hDir, - fileHandle: hFile, - sabView: state.sabFileBufView, - readOnly: create - ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), - deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) - }); - fh.releaseImplicitLocks = - (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) - || state.opfsFlags.defaultUnlockAsap; - if(0 /* this block is modelled after something wa-sqlite - does but it leads to immediate contention on journal files. */ - && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ - /* sqlite does not lock these files, so go ahead and grab an OPFS - lock. + }; - https://www.sqlite.org/uri.html - */ - fh.xLock = "xOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; - await getSyncHandle(fh,'xOpen'); - } - __openFiles[fid] = fh; - storeAndNotify(opName, 0); - }catch(e){ - wTimeEnd(); - error(opName,e); - state.s11n.storeException(1,e); - storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); - } - mTimeEnd(); - }, - xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xRead'); - let rc = 0, nRead; - const fh = __openFiles[fid]; - try{ - affirmLocked('xRead',fh); - wTimeStart('xRead'); - nRead = (await getSyncHandle(fh,'xRead')).read( - fh.sabView.subarray(0, n), - {at: Number(offset64)} - ); - wTimeEnd(); - if(nRead < n){/* Zero-fill remaining bytes */ - fh.sabView.fill(0, nRead, n); - rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; - } - }catch(e){ - if(undefined===nRead) wTimeEnd(); - error("xRead() failed",e,fh); - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); - } - await releaseImplicitLock(fh); - storeAndNotify('xRead',rc); - mTimeEnd(); - }, - xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ - mTimeStart('xSync'); - const fh = __openFiles[fid]; - let rc = 0; - if(!fh.readOnly && fh.syncHandle){ + /** + We track 2 different timers: the "metrics" timer records how much + time we spend performing work. The "wait" timer records how much + time we spend waiting on the underlying OPFS timer. See the calls + to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() + throughout this file to see how they're used. + */ + const __mTimer = Object.create(null); + __mTimer.op = undefined; + __mTimer.start = undefined; + const mTimeStart = (op)=>{ + __mTimer.start = performance.now(); + __mTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + ++metrics[op].count; + }; + const mTimeEnd = ()=>( + metrics[__mTimer.op].time += performance.now() - __mTimer.start + ); + const __wTimer = Object.create(null); + __wTimer.op = undefined; + __wTimer.start = undefined; + const wTimeStart = (op)=>{ + __wTimer.start = performance.now(); + __wTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + }; + const wTimeEnd = ()=>( + metrics[__wTimer.op].wait += performance.now() - __wTimer.start + ); + + /** + Gets set to true by the 'opfs-async-shutdown' command to quit the + wait loop. This is only intended for debugging purposes: we cannot + inspect this file's state while the tight waitLoop() is running and + need a way to stop that loop for introspection purposes. + */ + let flagAsyncShutdown = false; + + + /** + Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods + methods, as well as helpers like mkdir(). Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const vfsAsyncImpls = { + 'opfs-async-metrics': async ()=>{ + mTimeStart('opfs-async-metrics'); + metrics.dump(); + storeAndNotify('opfs-async-metrics', 0); + mTimeEnd(); + }, + 'opfs-async-shutdown': async ()=>{ + flagAsyncShutdown = true; + storeAndNotify('opfs-async-shutdown', 0); + }, + mkdir: async (dirname)=>{ + mTimeStart('mkdir'); + let rc = 0; + wTimeStart('mkdir'); try { - wTimeStart('xSync'); - await fh.syncHandle.flush(); + await getDirForFilename(dirname+"/filepart", true); }catch(e){ state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_FSYNC; + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('mkdir', rc); + mTimeEnd(); + }, + xAccess: async (filename)=>{ + mTimeStart('xAccess'); + /* OPFS cannot support the full range of xAccess() queries sqlite3 + calls for. We can essentially just tell if the file is + accessible, but if it is it's automatically writable (unless + it's locked, which we cannot(?) know without trying to open + it). OPFS does not have the notion of read-only. + + The return semantics of this function differ from sqlite3's + xAccess semantics because we are limited in what we can + communicate back to our synchronous communication partner: 0 = + accessible, non-0 means not accessible. + */ + let rc = 0; + wTimeStart('xAccess'); + try{ + const [dh, fn] = await getDirForFilename(filename); + await dh.getFileHandle(fn); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('xAccess', rc); + mTimeEnd(); + }, + xClose: async function(fid/*sqlite3_file pointer*/){ + const opName = 'xClose'; + mTimeStart(opName); + __implicitLocks.delete(fid); + const fh = __openFiles[fid]; + let rc = 0; + wTimeStart(opName); + if(fh){ + delete __openFiles[fid]; + await closeSyncHandle(fh); + if(fh.deleteOnClose){ + try{ await fh.dirHandle.removeEntry(fh.filenamePart) } + catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + } + }else{ + state.s11n.serialize(); + rc = state.sq3Codes.SQLITE_NOTFOUND; } wTimeEnd(); - } - storeAndNotify('xSync',rc); - mTimeEnd(); - }, - xTruncate: async function(fid/*sqlite3_file pointer*/,size){ - mTimeStart('xTruncate'); - let rc = 0; - const fh = __openFiles[fid]; - wTimeStart('xTruncate'); - try{ - affirmLocked('xTruncate',fh); - affirmNotRO('xTruncate', fh); - await (await getSyncHandle(fh,'xTruncate')).truncate(size); - }catch(e){ - error("xTruncate():",e,fh); - state.s11n.storeException(2,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); - } - await releaseImplicitLock(fh); - wTimeEnd(); - storeAndNotify('xTruncate',rc); - mTimeEnd(); - }, - xUnlock: async function(fid/*sqlite3_file pointer*/, + storeAndNotify(opName, rc); + mTimeEnd(); + }, + xDelete: async function(...args){ + mTimeStart('xDelete'); + const rc = await vfsAsyncImpls.xDeleteNoWait(...args); + storeAndNotify('xDelete', rc); + mTimeEnd(); + }, + xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ + /* The syncDir flag is, for purposes of the VFS API's semantics, + ignored here. However, if it has the value 0x1234 then: after + deleting the given file, recursively try to delete any empty + directories left behind in its wake (ignoring any errors and + stopping at the first failure). + + That said: we don't know for sure that removeEntry() fails if + the dir is not empty because the API is not documented. It has, + however, a "recursive" flag which defaults to false, so + presumably it will fail if the dir is not empty and that flag + is false. + */ + let rc = 0; + wTimeStart('xDelete'); + try { + while(filename){ + const [hDir, filenamePart] = await getDirForFilename(filename, false); + if(!filenamePart) break; + await hDir.removeEntry(filenamePart, {recursive}); + if(0x1234 !== syncDir) break; + recursive = false; + filename = getResolvedPath(filename, true); + filename.pop(); + filename = filename.join('/'); + } + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_DELETE; + } + wTimeEnd(); + return rc; + }, + xFileSize: async function(fid/*sqlite3_file pointer*/){ + mTimeStart('xFileSize'); + const fh = __openFiles[fid]; + let rc; + wTimeStart('xFileSize'); + try{ + affirmLocked('xFileSize',fh); + const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); + state.s11n.serialize(Number(sz)); + rc = 0; + }catch(e){ + state.s11n.storeException(2,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); + } + await releaseImplicitLock(fh); + wTimeEnd(); + storeAndNotify('xFileSize', rc); + mTimeEnd(); + }, + xLock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xUnlock'); - let rc = 0; - const fh = __openFiles[fid]; - if( state.sq3Codes.SQLITE_LOCK_NONE===lockType - && fh.syncHandle ){ - wTimeStart('xUnlock'); - try { await closeSyncHandle(fh) } - catch(e){ + mTimeStart('xLock'); + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + wTimeStart('xLock'); + try { + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); + fh.xLock = oldLockType; + } + wTimeEnd(); + } + storeAndNotify('xLock',rc); + mTimeEnd(); + }, + xOpen: async function(fid/*sqlite3_file pointer*/, filename, + flags/*SQLITE_OPEN_...*/, + opfsFlags/*OPFS_...*/){ + const opName = 'xOpen'; + mTimeStart(opName); + const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); + wTimeStart('xOpen'); + try{ + let hDir, filenamePart; + try { + [hDir, filenamePart] = await getDirForFilename(filename, !!create); + }catch(e){ + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); + mTimeEnd(); + wTimeEnd(); + return; + } + const hFile = await hDir.getFileHandle(filenamePart, {create}); + wTimeEnd(); + const fh = Object.assign(Object.create(null),{ + fid: fid, + filenameAbs: filename, + filenamePart: filenamePart, + dirHandle: hDir, + fileHandle: hFile, + sabView: state.sabFileBufView, + readOnly: create + ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), + deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) + }); + fh.releaseImplicitLocks = + (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) + || state.opfsFlags.defaultUnlockAsap; + if(0 /* this block is modelled after something wa-sqlite + does but it leads to immediate contention on journal files. */ + && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ + /* sqlite does not lock these files, so go ahead and grab an OPFS + lock. + + https://www.sqlite.org/uri.html + */ + fh.xLock = "xOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; + await getSyncHandle(fh,'xOpen'); + } + __openFiles[fid] = fh; + storeAndNotify(opName, 0); + }catch(e){ + wTimeEnd(); + error(opName,e); state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } + mTimeEnd(); + }, + xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xRead'); + let rc = 0, nRead; + const fh = __openFiles[fid]; + try{ + affirmLocked('xRead',fh); + wTimeStart('xRead'); + nRead = (await getSyncHandle(fh,'xRead')).read( + fh.sabView.subarray(0, n), + {at: Number(offset64)} + ); + wTimeEnd(); + if(nRead < n){/* Zero-fill remaining bytes */ + fh.sabView.fill(0, nRead, n); + rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; + } + }catch(e){ + if(undefined===nRead) wTimeEnd(); + error("xRead() failed",e,fh); + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); + } + await releaseImplicitLock(fh); + storeAndNotify('xRead',rc); + mTimeEnd(); + }, + xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ + mTimeStart('xSync'); + const fh = __openFiles[fid]; + let rc = 0; + if(!fh.readOnly && fh.syncHandle){ + try { + wTimeStart('xSync'); + await fh.syncHandle.flush(); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_FSYNC; + } + wTimeEnd(); + } + storeAndNotify('xSync',rc); + mTimeEnd(); + }, + xTruncate: async function(fid/*sqlite3_file pointer*/,size){ + mTimeStart('xTruncate'); + let rc = 0; + const fh = __openFiles[fid]; + wTimeStart('xTruncate'); + try{ + affirmLocked('xTruncate',fh); + affirmNotRO('xTruncate', fh); + await (await getSyncHandle(fh,'xTruncate')).truncate(size); + }catch(e){ + error("xTruncate():",e,fh); + state.s11n.storeException(2,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); + } + await releaseImplicitLock(fh); wTimeEnd(); - } - storeAndNotify('xUnlock',rc); - mTimeEnd(); - }, - xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xWrite'); - let rc; - const fh = __openFiles[fid]; - wTimeStart('xWrite'); - try{ - affirmLocked('xWrite',fh); - affirmNotRO('xWrite', fh); - rc = ( - n === (await getSyncHandle(fh,'xWrite')) - .write(fh.sabView.subarray(0, n), - {at: Number(offset64)}) - ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; - }catch(e){ - error("xWrite():",e,fh); - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); - } - await releaseImplicitLock(fh); - wTimeEnd(); - storeAndNotify('xWrite',rc); - mTimeEnd(); - } -}/*vfsAsyncImpls*/; - -const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; + storeAndNotify('xTruncate',rc); + mTimeEnd(); + }, + xUnlock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xUnlock'); + let rc = 0; + const fh = __openFiles[fid]; + if( state.sq3Codes.SQLITE_LOCK_NONE===lockType + && fh.syncHandle ){ + wTimeStart('xUnlock'); + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - rc.push(v); + wTimeEnd(); } + storeAndNotify('xUnlock',rc); + mTimeEnd(); + }, + xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xWrite'); + let rc; + const fh = __openFiles[fid]; + wTimeStart('xWrite'); + try{ + affirmLocked('xWrite',fh); + affirmNotRO('xWrite', fh); + rc = ( + n === (await getSyncHandle(fh,'xWrite')) + .write(fh.sabView.subarray(0, n), + {at: Number(offset64)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + error("xWrite():",e,fh); + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); + } + await releaseImplicitLock(fh); + wTimeEnd(); + storeAndNotify('xWrite',rc); + mTimeEnd(); } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; + }/*vfsAsyncImpls*/; + + const initS11n = ()=>{ + /** + ACHTUNG: this code is 100% duplicated in the other half of this + proxy! The documentation is maintained in the "synchronous half". + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; + }; + state.s11n.deserialize = function(clear=false){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); - } - }) - : ()=>{}; - - return state.s11n; -}/*initS11n()*/; - -const waitLoop = async function f(){ - const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ - const vi = vfsAsyncImpls[k]; - if(!vi) continue; - const o = Object.create(null); - opHandlers[state.opIds[k]] = o; - o.key = k; - o.f = vi; - } - /** - waitTime is how long (ms) to wait for each Atomics.wait(). - We need to wake up periodically to give the thread a chance - to do other things. If this is too high (e.g. 500ms) then - even two workers/tabs can easily run into locking errors. - */ - const waitTime = 100; - while(!flagAsyncShutdown){ - try { - if('timed-out'===Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, waitTime - )){ - await releaseImplicitLocks(); - continue; - } - const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); - Atomics.store(state.sabOPView, state.opIds.whichOp, 0); - const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); - const args = state.s11n.deserialize( - true /* clear s11n to keep the caller from confusing this with - an exception string written by the upcoming - operation */ - ) || []; - //warn("waitLoop() whichOp =",opId, hnd, args); - if(hnd.f) await hnd.f(...args); - else error("Missing callback for opId",opId); - }catch(e){ - error('in waitLoop():',e); - } - } -}; - -navigator.storage.getDirectory().then(function(d){ - const wMsg = (type)=>postMessage({type}); - state.rootDir = d; - self.onmessage = function({data}){ - switch(data.type){ - case 'opfs-async-init':{ - /* Receive shared state from synchronous partner */ - const opt = data.args; - state.littleEndian = opt.littleEndian; - state.asyncS11nExceptions = opt.asyncS11nExceptions; - state.verbose = opt.verbose ?? 1; - state.fileBufferSize = opt.fileBufferSize; - state.sabS11nOffset = opt.sabS11nOffset; - state.sabS11nSize = opt.sabS11nSize; - state.sabOP = opt.sabOP; - state.sabOPView = new Int32Array(state.sabOP); - state.sabIO = opt.sabIO; - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.opIds = opt.opIds; - state.sq3Codes = opt.sq3Codes; - state.opfsFlags = opt.opfsFlags; - Object.keys(vfsAsyncImpls).forEach((k)=>{ - if(!Number.isFinite(state.opIds[k])){ - toss("Maintenance required: missing state.opIds[",k,"]"); - } - }); - initS11n(); - metrics.reset(); - log("init state",state); - wMsg('opfs-async-inited'); - waitLoop(); - break; - } - case 'opfs-async-restart': - if(flagAsyncShutdown){ - warn("Restarting after opfs-async-shutdown. Might or might not work."); - flagAsyncShutdown = false; - waitLoop(); + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; } - break; - case 'opfs-async-metrics': - metrics.dump(); - break; + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; + + return state.s11n; + }/*initS11n()*/; + + const waitLoop = async function f(){ + const opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } + /** + waitTime is how long (ms) to wait for each Atomics.wait(). + We need to wake up periodically to give the thread a chance + to do other things. If this is too high (e.g. 500ms) then + even two workers/tabs can easily run into locking errors. + */ + const waitTime = 100; + while(!flagAsyncShutdown){ + try { + if('timed-out'===Atomics.wait( + state.sabOPView, state.opIds.whichOp, 0, waitTime + )){ + await releaseImplicitLocks(); + continue; + } + const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); + Atomics.store(state.sabOPView, state.opIds.whichOp, 0); + const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + const args = state.s11n.deserialize( + true /* clear s11n to keep the caller from confusing this with + an exception string written by the upcoming + operation */ + ) || []; + //warn("waitLoop() whichOp =",opId, hnd, args); + if(hnd.f) await hnd.f(...args); + else error("Missing callback for opId",opId); + }catch(e){ + error('in waitLoop():',e); + } } }; - wMsg('opfs-async-loaded'); -}).catch((e)=>error("error initializing OPFS asyncer:",e)); + + navigator.storage.getDirectory().then(function(d){ + state.rootDir = d; + self.onmessage = function({data}){ + switch(data.type){ + case 'opfs-async-init':{ + /* Receive shared state from synchronous partner */ + const opt = data.args; + state.littleEndian = opt.littleEndian; + state.asyncS11nExceptions = opt.asyncS11nExceptions; + state.verbose = opt.verbose ?? 1; + state.fileBufferSize = opt.fileBufferSize; + state.sabS11nOffset = opt.sabS11nOffset; + state.sabS11nSize = opt.sabS11nSize; + state.sabOP = opt.sabOP; + state.sabOPView = new Int32Array(state.sabOP); + state.sabIO = opt.sabIO; + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.opIds = opt.opIds; + state.sq3Codes = opt.sq3Codes; + state.opfsFlags = opt.opfsFlags; + Object.keys(vfsAsyncImpls).forEach((k)=>{ + if(!Number.isFinite(state.opIds[k])){ + toss("Maintenance required: missing state.opIds[",k,"]"); + } + }); + initS11n(); + metrics.reset(); + log("init state",state); + wPost('opfs-async-inited'); + waitLoop(); + break; + } + case 'opfs-async-restart': + if(flagAsyncShutdown){ + warn("Restarting after opfs-async-shutdown. Might or might not work."); + flagAsyncShutdown = false; + waitLoop(); + } + break; + case 'opfs-async-metrics': + metrics.dump(); + break; + } + }; + wPost('opfs-async-loaded'); + }).catch((e)=>error("error initializing OPFS asyncer:",e)); +}/*installAsyncProxy()*/; +if(!self.SharedArrayBuffer){ + wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", + "The server must emit the COOP/COEP response headers to enable that."); +}else if(!self.Atomics){ + wPost('opfs-unavailable', "Missing Atomics API.", + "The server must emit the COOP/COEP response headers to enable that."); +}else if(!self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + wPost('opfs-unavailable',"Missing required OPFS APIs."); +}else{ + installAsyncProxy(self); +} diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 5bb5959396..2fc6752694 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1784,13 +1784,12 @@ self.sqlite3InitModule = sqlite3InitModule; .t({ name: 'OPFS sanity checks', test: async function(sqlite3){ - const opfs = sqlite3.opfs; const filename = 'sqlite3-tester1.db'; const pVfs = capi.sqlite3_vfs_find('opfs'); T.assert(pVfs); const unlink = (fn=filename)=>wasm.sqlite3_wasm_vfs_unlink(pVfs,fn); unlink(); - let db = new opfs.OpfsDb(filename); + let db = new sqlite3.oo1.OpfsDb(filename); try { db.exec([ 'create table p(a);', @@ -1798,7 +1797,7 @@ self.sqlite3InitModule = sqlite3InitModule; ]); T.assert(3 === db.selectValue('select count(*) from p')); db.close(); - db = new opfs.OpfsDb(filename); + db = new sqlite3.oo1.OpfsDb(filename); db.exec('insert into p(a) values(4),(5),(6)'); T.assert(6 === db.selectValue('select count(*) from p')); }finally{ @@ -1806,8 +1805,9 @@ self.sqlite3InitModule = sqlite3InitModule; unlink(); } - if(1){ + if(sqlite3.opfs){ // Sanity-test sqlite3_wasm_vfs_create_file()... + const opfs = sqlite3.opfs; const fSize = 1379; let sh; try{ @@ -1824,20 +1824,20 @@ self.sqlite3InitModule = sqlite3InitModule; if(sh) await sh.close(); unlink(); } - } - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } } }/*OPFS sanity checks*/) ;/* end OPFS tests */ diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index e19f6a8da6..d5aa51dc93 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -33,9 +33,10 @@ with unlock-asap=0-1.

    Achtung: if it does not start to do anything within a couple of - seconds, check the dev console: Chrome often fails with "cannot allocate - WasmMemory" at startup. Closing and re-opening the tab usually resolves - it. + seconds, check the dev console: Chrome sometimes fails to load + the wasm module due to "cannot allocate WasmMemory." Closing and + re-opening the tab usually resolves it, but sometimes restarting + the browser is required.

    diff --git a/manifest b/manifest index 01aaf2a316..4dc42b8b8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexplicit\swarning\sabout\sthe\scurrent\sAPI-instability\sof\sthe\ssqlite3.opfs\snamespace,\swhich\smay\sneed\sto\sbe\seliminated\sbased\son\sre-thinking\sof\show\sthe\sOPFS\ssqlite3_vfs\sis\sregistered.\sComment\schanges\sonly\s-\sno\scode. -D 2022-11-29T02:23:12.943 +C Internal\srestructuring\sof\sthe\sOPFS\ssqlite3_vfs\sin\sorder\sto\sfacilitate\scertain\sexperimentation\sand\simprove\serror\sreporting/hints\sif\sit\scannot\sbe\sactivated.\sDeprecate\sthe\sname\ssqlite3.opfs.OpfsDb,\spreferring\ssqlite3.oo1.OpfsDb\sfor\sconsistency\swith\sJsStorageDb\sand\sany\sfuture\sDB\ssubclasses. +D 2022-11-29T05:25:08.036 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,11 +505,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js 3cdae7e98c500f89f9468a260e2a0e1b528c845a107bf72d368e5222769214d3 +F ext/wasm/api/sqlite3-api-opfs.js 583650ffdc1452496df6b9459d018fa2aede221ae6ea0cbbbe83bd2e1bdba966 F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 798383f6b46fd5dac122d6e35962d25b10401ddb825b5c66df1d21e6b1d8aacc +F ext/wasm/api/sqlite3-opfs-async-proxy.js b5dd7eda8e74e07453457925a0dd793d7785da720954e0e37e847c5c6e4d9526 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,8 +554,8 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js 3b91f192c159088004fba6fe3441edea58421a8b88bccf3dd20978a077648d19 -F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c +F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa +F ext/wasm/tests/opfs/concurrency/index.html 2b1cda51d6c786102875a28eba22f0da3eecb732a5e677b0d1ecdb53546d1a62 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 -R 3870d04bfd54da096e986662fe29b1c8 +P 0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 +R 115b7898d7b2ce79a9261f36a9b959d1 U stephan -Z ad4a8c5f45a34a10f588c0c6dc455846 +Z f4ff31d5e2499971cf67cb62dbdd0ac3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 361de4178e..fb6fdac183 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 \ No newline at end of file +0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 \ No newline at end of file From ceedef888f38d172c203b1c7bf5265f4ee608640 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 06:09:32 +0000 Subject: [PATCH 62/72] Minor internal cleanups and docs in the OPFS sqlite3_vfs. FossilOrigin-Name: 61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 --- ext/wasm/api/sqlite3-api-opfs.js | 4 +++- ext/wasm/api/sqlite3-opfs-async-proxy.js | 27 ++++++++++++++++++------ manifest | 14 ++++++------ manifest.uuid | 2 +- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index cef15305d4..de7500afa3 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -290,7 +290,9 @@ const installOpfsVfs = function callee(options){ The size of the block in our SAB for serializing arguments and result values. Needs to be large enough to hold serialized values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. */ state.sabS11nSize = opfsVfs.$mxPathname * 2; /** diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 1ba6e9bdbb..a80eeb2903 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -239,15 +239,26 @@ const installAsyncProxy = function(self){ between locking-related failures and other types, noting that we cannot currently do so because createSyncAccessHandle() does not define its exceptions in the required level of detail. + + 2022-11-29: according to: + + https://github.com/whatwg/fs/pull/21 + + NoModificationAllowedError will be the standard exception thrown + when acquisition of a sync access handle fails due to a locking + error. As of this writing, that error type is not visible in the + dev console in Chrome v109, nor is it documented in MDN, but an + error with that "name" property is being thrown from the OPFS + layer. */ class GetSyncHandleError extends Error { constructor(errorObject, ...msg){ - super(); - this.error = errorObject; - this.message = [ - ...msg, ': Original exception ['+errorObject.name+']:', + super([ + ...msg, ': '+errorObject.name+':', errorObject.message - ].join(' '); + ].join(' '), { + cause: errorObject + }); this.name = 'GetSyncHandleError'; } }; @@ -259,8 +270,12 @@ const installAsyncProxy = function(self){ distinguish that from other errors. This approach is highly questionable. + + Note that even if we return SQLITE_IOERR_LOCK from here, + it bubbles up to the client as a plain I/O error. */ - return (e instanceof GetSyncHandleError) + return (e instanceof GetSyncHandleError + && e.cause.name==='NoModificationAllowedError') ? state.sq3Codes.SQLITE_IOERR_LOCK : rc; }else{ diff --git a/manifest b/manifest index 4dc42b8b8c..51b58eba3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\srestructuring\sof\sthe\sOPFS\ssqlite3_vfs\sin\sorder\sto\sfacilitate\scertain\sexperimentation\sand\simprove\serror\sreporting/hints\sif\sit\scannot\sbe\sactivated.\sDeprecate\sthe\sname\ssqlite3.opfs.OpfsDb,\spreferring\ssqlite3.oo1.OpfsDb\sfor\sconsistency\swith\sJsStorageDb\sand\sany\sfuture\sDB\ssubclasses. -D 2022-11-29T05:25:08.036 +C Minor\sinternal\scleanups\sand\sdocs\sin\sthe\sOPFS\ssqlite3_vfs. +D 2022-11-29T06:09:32.898 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,11 +505,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js 583650ffdc1452496df6b9459d018fa2aede221ae6ea0cbbbe83bd2e1bdba966 +F ext/wasm/api/sqlite3-api-opfs.js ee97811004fd0da9311527293b519543fbb0c6b0c9beaa9b6704400bfd74d150 F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js b5dd7eda8e74e07453457925a0dd793d7785da720954e0e37e847c5c6e4d9526 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 4b8973711ae4825bb047da932b9046b4e8f678fcea759158caebc5821a655ac7 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 -R 115b7898d7b2ce79a9261f36a9b959d1 +P 0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 +R e8239b21ab627d1eba84cc5be45ed7a1 U stephan -Z f4ff31d5e2499971cf67cb62dbdd0ac3 +Z 1147c5417905a9e8323d8798a4fd57dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fb6fdac183..85d7059802 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 \ No newline at end of file +61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 \ No newline at end of file From a302e6847643c24d331e1f1be5275bf8ec38be46 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 06:56:09 +0000 Subject: [PATCH 63/72] Rename one instance of opfs.OpfsDb to oo1.OpfsDb, as per [0c5c51f4fb04]. FossilOrigin-Name: 75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 --- ext/wasm/tests/opfs/concurrency/worker.js | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 95fefa1954..5d28bedee0 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -46,7 +46,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.opfs.OpfsDb({ + db = new sqlite3.oo1.OpfsDb({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c' }); diff --git a/manifest b/manifest index 51b58eba3a..cef62cf1b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\scleanups\sand\sdocs\sin\sthe\sOPFS\ssqlite3_vfs. -D 2022-11-29T06:09:32.898 +C Rename\sone\sinstance\sof\sopfs.OpfsDb\sto\soo1.OpfsDb,\sas\sper\s[0c5c51f4fb04]. +D 2022-11-29T06:56:09.516 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -557,7 +557,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa F ext/wasm/tests/opfs/concurrency/index.html 2b1cda51d6c786102875a28eba22f0da3eecb732a5e677b0d1ecdb53546d1a62 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a -F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 +F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 -R e8239b21ab627d1eba84cc5be45ed7a1 +P 61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 +R ccbd533ba6a1af3ea32f1c34f9412b94 U stephan -Z 1147c5417905a9e8323d8798a4fd57dd +Z 5d4b5205e72d1fad1817c77ba7b7ffb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 85d7059802..08f00fa593 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 \ No newline at end of file +75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 \ No newline at end of file From 3c3896c9d68f1003edbd02f0657e875ae00dff5e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Nov 2022 11:28:45 +0000 Subject: [PATCH 64/72] Fix legacy build system main.mk to include basexx.c in testfixture builds. FossilOrigin-Name: 8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 --- main.mk | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/main.mk b/main.mk index 76c62fcecd..ccd5b6f5b0 100644 --- a/main.mk +++ b/main.mk @@ -363,6 +363,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/basexx.c \ $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/cksumvfs.c \ $(TOP)/ext/misc/closure.c \ diff --git a/manifest b/manifest index cef62cf1b4..becd51a6d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sone\sinstance\sof\sopfs.OpfsDb\sto\soo1.OpfsDb,\sas\sper\s[0c5c51f4fb04]. -D 2022-11-29T06:56:09.516 +C Fix\slegacy\sbuild\ssystem\smain.mk\sto\sinclude\sbasexx.c\sin\stestfixture\sbuilds. +D 2022-11-29T11:28:45.200 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -563,7 +563,7 @@ F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c7974 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk fd90f1bd90bd4070c9af7e9c8396bd0cf4208a0186c8d18fa6e2609dff447d5d +F main.mk 5ce166d63fb99df9821f262fb1777fd5eb7b6c9396e867a6b0b2761f35f8238a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 -R ccbd533ba6a1af3ea32f1c34f9412b94 -U stephan -Z 5d4b5205e72d1fad1817c77ba7b7ffb7 +P 75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 +R 5b4d2911b428490bdd01585b55a3d462 +U dan +Z d91547f2ddaedf2a7b8258684ff0e02a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 08f00fa593..989d35069a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 \ No newline at end of file +8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 \ No newline at end of file From 7741f3457a3c3e6785d987fe1be46351759851a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Nov 2022 17:52:04 +0000 Subject: [PATCH 65/72] Change the handling of hwtime.h to make it easier to compile performance measurement builds that make use of hwtime.h. This should not affect productions builds. FossilOrigin-Name: f64a224244743ab121371abd516fccbfc93c110e0952211764bd1b217e792c1b --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os_common.h | 6 ------ src/sqliteInt.h | 4 ++++ src/util.c | 7 +++++++ src/vdbe.c | 10 ---------- 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index becd51a6d3..c06c8f707c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slegacy\sbuild\ssystem\smain.mk\sto\sinclude\sbasexx.c\sin\stestfixture\sbuilds. -D 2022-11-29T11:28:45.200 +C Change\sthe\shandling\sof\shwtime.h\sto\smake\sit\seasier\sto\scompile\sperformance\nmeasurement\sbuilds\sthat\smake\suse\sof\shwtime.h.\s\sThis\sshould\snot\saffect\nproductions\sbuilds. +D 2022-11-29T17:52:04.105 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/mutex_w32.c caa50e1c0258ac4443f52e00fe8aaea73b6d0728bd8856bedfff822cae4185 F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 F src/os.c 81c9c1c52eab711e27e33fd51fe5788488d3a02bc1a71439857abbee5d0d2c97 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 -F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 +F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 0e59600d25b72034c7666b8b7dcc527f039b5d9c16f24a7eca4c08c66f63c364 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c @@ -649,7 +649,7 @@ F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063 F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f +F src/sqliteInt.h a8d507577ca5e6581abb65c30497f7b8cfc7c9feb0d768a081c1e4a6adfcd061 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -714,9 +714,9 @@ F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f +F src/util.c 4e42c338d982e3f139004467bf0f2e9d679d519c7c75718fcf4ff1811401b03c F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c c2d6d0c0c343d8ebffef996c73cbb69e337225f757fea7fe5c0e3ea14662adec +F src/vdbe.c 00648bd76fb2145c2d890312112bda446569da64ca70aaf1f2282cc6c2b22d14 F src/vdbe.h 58675f47dcf3105bab182c3ad3726efd60ffd003e954386904ac9107d0d2b743 F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f F src/vdbeapi.c 1e8713d0b653acb43cd1bdf579c40e005c4844ea90f414f065946a83db3c27fb @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 -R 5b4d2911b428490bdd01585b55a3d462 -U dan -Z d91547f2ddaedf2a7b8258684ff0e02a +P 8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 +R 427322317ebe04d9616d118c250b6e39 +U drh +Z a5571007571b0b63420d1f8b08046ddf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 989d35069a..94da90e067 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 \ No newline at end of file +f64a224244743ab121371abd516fccbfc93c110e0952211764bd1b217e792c1b \ No newline at end of file diff --git a/src/os_common.h b/src/os_common.h index 1ed4d7a8e1..5b532af0ac 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -35,12 +35,6 @@ */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - static sqlite_uint64 g_start; static sqlite_uint64 g_elapsed; #define TIMER_START g_start=sqlite3Hwtime() diff --git a/src/sqliteInt.h b/src/sqliteInt.h index db65082bbc..7d7e5b56f0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5535,4 +5535,8 @@ const char **sqlite3CompileOptions(int *pnOpt); int sqlite3KvvfsInit(void); #endif +#if defined(VDBE_PROFILE) || defined(SQLITE_PERFORMANCE_TRACE) +sqlite3_uint64 sqlite3Hwtime(void); +#endif + #endif /* SQLITEINT_H */ diff --git a/src/util.c b/src/util.c index 32e9c27786..7a58fc8764 100644 --- a/src/util.c +++ b/src/util.c @@ -1713,3 +1713,10 @@ int sqlite3VListNameToNum(VList *pIn, const char *zName, int nName){ }while( i Date: Tue, 29 Nov 2022 18:00:01 +0000 Subject: [PATCH 66/72] OPFS speedtest1: hide a currently-broken/to-fix WASMFS-build link. FossilOrigin-Name: cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c --- ext/wasm/speedtest1-worker.html | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 5638ce3b50..9fcd29ea72 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -40,7 +40,7 @@ → speedtest1 - → speedtest1-wasmfs → Date: Tue, 29 Nov 2022 18:28:40 +0000 Subject: [PATCH 67/72] sqlite3.oo1.OpfsDb: default to journal_mode=persist, as current tests show it to be marginally faster than truncate/delete in Chrome v109. Also increase default busy_timeout from 2 seconds to 3, admittedly on a whim. FossilOrigin-Name: d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 --- ext/wasm/api/sqlite3-api-opfs.js | 10 ++++++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index de7500afa3..3099cb486d 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1216,12 +1216,14 @@ const installOpfsVfs = function callee(options){ sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, [ - /* Truncate journal mode is faster than delete or wal for - this vfs, per speedtest1. */ - "pragma journal_mode=truncate;", + /* Truncate journal mode is faster than delete for + this vfs, per speedtest1. That gap seems to have closed with + Chome version 108 or 109, but "persist" is very roughly 5-6% + faster than truncate in initial tests. */ + "pragma journal_mode=persist;", /* Set a default busy-timeout handler to help OPFS dbs deal with multi-tab/multi-worker contention. */ - "pragma busy_timeout=2000;", + "pragma busy_timeout=3000;", /* This vfs benefits hugely from cache on moderate/large speedtest1 --size 50 and --size 100 workloads. We currently diff --git a/manifest b/manifest index 0721c80593..e7c5ce5e81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sspeedtest1:\shide\sa\scurrently-broken/to-fix\sWASMFS-build\slink. -D 2022-11-29T18:00:01.024 +C sqlite3.oo1.OpfsDb:\sdefault\sto\sjournal_mode=persist,\sas\scurrent\stests\sshow\sit\sto\sbe\smarginally\sfaster\sthan\struncate/delete\sin\sChrome\sv109.\sAlso\sincrease\sdefault\sbusy_timeout\sfrom\s2\sseconds\sto\s3,\sadmittedly\son\sa\swhim. +D 2022-11-29T18:28:40.756 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js ee97811004fd0da9311527293b519543fbb0c6b0c9beaa9b6704400bfd74d150 +F ext/wasm/api/sqlite3-api-opfs.js 2f99c9b2c3c0f53bf40153de20db350378deed825ec0d5cf8f449e9e0081f08b F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f64a224244743ab121371abd516fccbfc93c110e0952211764bd1b217e792c1b -R 9e4ddb7294c8c5e99856ae19fd524910 +P cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c +R f012421262a97adb4d00809196a366a2 U stephan -Z 303f08b9a1a9cd719a67e29d0dc3ab85 +Z e12ff5fd260034aa46c1df81ecc7ab20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5ae68dd9ad..43e23def0d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c \ No newline at end of file +d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 \ No newline at end of file From d0945f4638f342daf4c84ab5a03449137435dd36 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 03:08:50 +0000 Subject: [PATCH 68/72] Doc and logging text tweaks in the OPFS async proxy and test app. FossilOrigin-Name: 7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 13 +++++++------ ext/wasm/tests/opfs/concurrency/index.html | 4 +--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index a80eeb2903..da75f139c9 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -287,11 +287,12 @@ const installAsyncProxy = function(self){ handle object (which must be a valid handle object, as created by xOpen()), lazily opening it if needed. - In order to help alleviate cross-tab contention for a dabase, - if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to 3 times. If acquisition - still fails at that point it will give up and propagate the - exception. + In order to help alleviate cross-tab contention for a dabase, if + an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to some fixed number of + times. If acquisition still fails at that point it will give up + and propagate the exception. Client-level code will see that as + an I/O error. */ const getSyncHandle = async (fh,opName)=>{ if(!fh.syncHandle){ @@ -323,7 +324,7 @@ const installAsyncProxy = function(self){ 'in',performance.now() - t,'ms'); if(!fh.xLock){ __implicitLocks.add(fh.fid); - log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); + log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs); } } return fh.syncHandle; diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index d5aa51dc93..9a826c6e13 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -18,9 +18,7 @@

    OPFS concurrency tester using multiple independent Workers. - Disclaimer: concurrency in OPFS is currently a pain point - and timing/concurrency mitigation in this environment is - highly unpredictable! + Disclaimer: concurrency in OPFS is currently a pain point!

    URL flags: pass a number of workers using diff --git a/manifest b/manifest index e7c5ce5e81..ecfa59693e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C sqlite3.oo1.OpfsDb:\sdefault\sto\sjournal_mode=persist,\sas\scurrent\stests\sshow\sit\sto\sbe\smarginally\sfaster\sthan\struncate/delete\sin\sChrome\sv109.\sAlso\sincrease\sdefault\sbusy_timeout\sfrom\s2\sseconds\sto\s3,\sadmittedly\son\sa\swhim. -D 2022-11-29T18:28:40.756 +C Doc\sand\slogging\stext\stweaks\sin\sthe\sOPFS\sasync\sproxy\sand\stest\sapp. +D 2022-11-30T03:08:50.289 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -509,7 +509,7 @@ F ext/wasm/api/sqlite3-api-opfs.js 2f99c9b2c3c0f53bf40153de20db350378deed825ec0d F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 4b8973711ae4825bb047da932b9046b4e8f678fcea759158caebc5821a655ac7 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 315582a4fe3b87ceaff69ad4a67d68a350b8014ac94d6a584d09ea922453c983 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa -F ext/wasm/tests/opfs/concurrency/index.html 2b1cda51d6c786102875a28eba22f0da3eecb732a5e677b0d1ecdb53546d1a62 +F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c -R f012421262a97adb4d00809196a366a2 +P d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 +R 05044488394ac3d7b358270198570ce0 U stephan -Z e12ff5fd260034aa46c1df81ecc7ab20 +Z cb806f606aa315e69c2e3fceeb852a45 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 43e23def0d..12e29fe3c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 \ No newline at end of file +7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b \ No newline at end of file From ad4f7828153e6b80c0fceabb1a9ece702172b836 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 05:27:36 +0000 Subject: [PATCH 69/72] Refactor a significant chunk of the OPFS sqlite3_vfs init code into sqlite3.VfsHelper, and internal-use-only API encapsulating code relevant to creating new VFSes in JS. Intended to assist in pending experimentation with an alternative OPFS VFS. FossilOrigin-Name: e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 --- ext/wasm/GNUmakefile | 1 + ext/wasm/api/README.md | 27 ++-- ext/wasm/api/extern-post-js.js | 12 +- ext/wasm/api/sqlite3-api-cleanup.js | 2 - ext/wasm/api/sqlite3-api-opfs.js | 108 ++------------ ext/wasm/api/sqlite3-api-prologue.js | 17 ++- ext/wasm/api/sqlite3-vfs-helper.js | 209 +++++++++++++++++++++++++++ ext/wasm/tester1.c-pp.js | 3 + manifest | 25 ++-- manifest.uuid | 2 +- 10 files changed, 280 insertions(+), 126 deletions(-) create mode 100644 ext/wasm/api/sqlite3-vfs-helper.js diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 38c1cdaca8..3c6b3c9a15 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -287,6 +287,7 @@ sqlite3-api.jses += $(dir.common)/whwasmutil.js sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js sqlite3-api.jses += $(sqlite3-api-build-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index d33c4a5831..37d986584e 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -23,13 +23,13 @@ The overall idea is that the following files get concatenated together, in the listed order, the resulting file is loaded by a browser client: -- `sqlite3-api-prologue.js`\ +- **`sqlite3-api-prologue.js`**\ Contains the initial bootstrap setup of the sqlite3 API objects. This is exposed as a function, rather than objects, so that the next step can pass in a config object which abstracts away parts of the WASM environment, to facilitate plugging it in to arbitrary WASM toolchains. -- `../common/whwasmutil.js`\ +- **`../common/whwasmutil.js`**\ A semi-third-party collection of JS/WASM utility code intended to replace much of the Emscripten glue. The sqlite3 APIs internally use these APIs instead of their Emscripten counterparts, in order to be @@ -38,47 +38,52 @@ browser client: toolchains. It is "semi-third-party" in that it was created in order to support this tree but is standalone and maintained together with... -- `../jaccwabyt/jaccwabyt.js`\ +- **`../jaccwabyt/jaccwabyt.js`**\ Another semi-third-party API which creates bindings between JS and C structs, such that changes to the struct state from either JS or C are visible to the other end of the connection. This is also an independent spinoff project, conceived for the sqlite3 project but maintained separately. -- `sqlite3-api-glue.js`\ +- **`sqlite3-api-glue.js`**\ Invokes functionality exposed by the previous two files to flesh out low-level parts of `sqlite3-api-prologue.js`. Most of these pieces related to the `sqlite3.capi.wasm` object. -- `sqlite3-api-build-version.js`\ +- **`sqlite3-api-build-version.js`**\ Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- `sqlite3-api-oo1.js`\ +- **`sqlite3-vfs-helper.js`**\ + This internal-use-only file installs `sqlite3.VfsHelper` for use by + `sqlite3-api-*.js` files which create `sqlite3_vfs` implemenations. + `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after + the library is finished initializing. +- **`sqlite3-api-oo1.js`**\ Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. That said, it is not a "required component" and can be elided from builds which do not want it. -- `sqlite3-api-worker1.js`\ +- **`sqlite3-api-worker1.js`**\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - `sqlite3-worker1.js`\ + - **`sqlite3-worker1.js`**\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module and runs the Worker #1 API which is implemented in `sqlite3-api-worker1.js`. - - `sqlite3-worker1-promiser.js`\ + - **`sqlite3-worker1-promiser.js`**\ Is likewise not part of the amalgamated sources and provides a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- `sqlite3-api-opfs.js`\ +- **`sqlite3-api-opfs.js`**\ is an sqlite3 VFS implementation which supports Google Chrome's Origin-Private FileSystem (OPFS) as a storage layer to provide persistent storage for database files in a browser. It requires... - - `sqlite3-opfs-async-proxy.js`\ + - **`sqlite3-opfs-async-proxy.js`**\ is the asynchronous backend part of the OPFS proxy. It speaks directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index cace6ed51c..2258697944 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -60,7 +60,7 @@ const toExportForES6 = initModuleState.sqlite3Dir = li.join('/') + '/'; } - self.sqlite3InitModule = (...args)=>{ + self.sqlite3InitModule = function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",self.location); return originalInit(...args).then((EmscriptenModule)=>{ if(self.window!==self && @@ -76,10 +76,12 @@ const toExportForES6 = Emscripten details. */ return EmscriptenModule; } - EmscriptenModule.sqlite3.scriptInfo = initModuleState; - //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo); - const f = EmscriptenModule.sqlite3.asyncPostInit; - delete EmscriptenModule.sqlite3.asyncPostInit; + const s = EmscriptenModule.sqlite3; + s.scriptInfo = initModuleState; + //console.warn("sqlite3.scriptInfo =",s.scriptInfo); + if(ff.__isUnderTest) s.__isUnderTest = true; + const f = s.asyncPostInit; + delete s.asyncPostInit; return f(); }).catch((e)=>{ console.error("Exception loading sqlite3 module:",e); diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index bef4d91d70..0ec0fbfbe3 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -58,8 +58,6 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build self.S = sqlite3; } - /* Clean up temporary references to our APIs... */ - delete sqlite3.util /* arguable, but these are (currently) internal-use APIs */; Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to pass the sqlite3 object off to the client. */; }else{ diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 3099cb486d..5eb3b22b37 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -129,7 +129,7 @@ const installOpfsVfs = function callee(options){ const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); const error = (...args)=>logImpl(0, ...args); - const toss = function(...args){throw new Error(args.join(' '))}; + const toss = sqlite3.util.toss; const capi = sqlite3.capi; const wasm = sqlite3.wasm; const sqlite3_vfs = capi.sqlite3_vfs; @@ -191,6 +191,8 @@ const installOpfsVfs = function callee(options){ s.count = s.time = 0; } }/*metrics*/; + const opfsVfs = new sqlite3_vfs(); + const opfsIoMethods = new sqlite3_io_methods(); const promiseReject = function(err){ opfsVfs.dispose(); return promiseReject_(err); @@ -213,8 +215,6 @@ const installOpfsVfs = function callee(options){ ? new sqlite3_vfs(pDVfs) : null /* dVfs will be null when sqlite3 is built with SQLITE_OS_OTHER. */; - const opfsVfs = new sqlite3_vfs(); - const opfsIoMethods = new sqlite3_io_methods(); opfsVfs.$iVersion = 2/*yes, two*/; opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; opfsVfs.$mxPathname = 1024/*sure, why not?*/; @@ -633,77 +633,6 @@ const installOpfsVfs = function callee(options){ */ const __openFiles = Object.create(null); - /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error (e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc.). - - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. - - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - */ - const installMethod = function callee(tgt, name, func){ - if(!(tgt instanceof sqlite3.StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); - } - const memKey = tgt.memberKey(name); - const fProxy = 0 - /** This middle-man proxy is only for use during development, to - confirm that we always pass the proper number of - arguments. We know that the C-level code will always use the - correct argument count. */ - ? callee.argcProxy(func, sigN) - : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - const opTimer = Object.create(null); opTimer.op = undefined; opTimer.start = undefined; @@ -729,7 +658,11 @@ const installOpfsVfs = function callee(options){ success) release a sync-handle for it, but doing so would involve an inherent race condition. For the time being, pending a better solution, we simply report whether the - given pFile instance has a lock. + given pFile is open. + + FIXME: we need to ask the async half whether a lock is + held, as it's possible (since long after this method was + implemented) that we do not hold a lock on an OPFS file. */ const f = __openFiles[pFile]; wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); @@ -948,14 +881,6 @@ const installOpfsVfs = function callee(options){ }; } - /* Install the vfs/io_methods into their C-level shared instances... */ - for(let k of Object.keys(ioSyncWrappers)){ - installMethod(opfsIoMethods, k, ioSyncWrappers[k]); - } - for(let k of Object.keys(vfsSyncWrappers)){ - installMethod(opfsVfs, k, vfsSyncWrappers[k]); - } - /** Expects an OPFS file path. It gets resolved, such that ".." components are properly expanded, and returned. If the 2nd arg @@ -1095,8 +1020,9 @@ const installOpfsVfs = function callee(options){ Irrevocably deletes _all_ files in the current origin's OPFS. Obviously, this must be used with great caution. It may throw an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which it will throw - are not documented (so we cannot tell you what they are). + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). */ opfsUtil.rmfr = async function(){ const dir = opfsUtil.rootDirectory, opt = {recurse: true}; @@ -1320,14 +1246,10 @@ const installOpfsVfs = function callee(options){ and has finished initializing, so the real work can begin...*/ try { - const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0); - if(rc){ - toss("sqlite3_vfs_register(OPFS) failed with rc",rc); - } - if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){ - toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); - } - capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); + sqlite3.VfsHelper.installVfs({ + io: {struct: opfsIoMethods, methods: ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: vfsSyncWrappers} + }); state.sabOPView = new Int32Array(state.sabOP); state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 726f47e18d..6d39581965 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -612,8 +612,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( isBindableTypedArray, isInt32, isSQLableTypedArray, isTypedArray, typedArrayToString, - isUIThread: ()=>'undefined'===typeof WorkerGlobalScope, + isUIThread: ()=>(self.window===self && !!self.document), + // is this true for ESM?: 'undefined'===typeof WorkerGlobalScope isSharedTypedArray, + toss: function(...args){throw new Error(args.join(' '))}, + toss3, typedArrayPart }; @@ -1460,7 +1463,17 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( //let p = lip.shift(); //while(lip.length) p = p.then(lip.shift()); //return p.then(()=>sqlite3); - return Promise.all(lip).then(()=>sqlite3); + return Promise.all(lip).then(()=>{ + if(!sqlite3.__isUnderTest){ + /* Delete references to internal-only APIs which are used by + some initializers. Retain them when running in test mode + so that we can add tests for them. */ + delete sqlite3.util; + delete sqlite3.VfsHelper; + delete sqlite3.StructBinder; + } + return sqlite3; + }); }, /** scriptInfo ideally gets injected into this object by the diff --git a/ext/wasm/api/sqlite3-vfs-helper.js b/ext/wasm/api/sqlite3-vfs-helper.js new file mode 100644 index 0000000000..9a15dd85fd --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-helper.js @@ -0,0 +1,209 @@ +/* +** 2022-11-30 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ + +/** + This file installs sqlite.VfsHelper, an object which exists + to assist in the creation of JavaScript implementations of + sqlite3_vfs. It is NOT part of the public API, and is an + internal implemenation detail for use in this project's + own development of VFSes. It may be exposed to clients + at some point, provided there is value in doing so. +*/ +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss; + const vh = Object.create(null); + + /** + Does nothing more than holds a permanent reference to each + argument. This is useful in some cases to ensure that, e.g., a + custom sqlite3_io_methods instance does not get + garbage-collected. + + Returns this object. + */ + vh.holdReference = function(...args){ + for(const v of args) this.refs.add(v); + return vh; + }.bind({refs: new Set}); + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error, e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc. + + If applyArgcCheck is true then each method gets wrapped in a + proxy which asserts that it is passed the expected number of + arguments, throwing if the argument count does not match + expectations. That is only recommended for dev-time usage for + sanity checking. Once a VFS implementation is known to be + working, it is a given that the C API will never call it with the + wrong argument count. + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + + If tgt.ondispose is set before this is called then it _must_ + be an array, to which this function will append entries. + */ + vh.installMethod = function callee(tgt, name, func, + applyArgcCheck=callee.installMethodArgcCheck){ + if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + /* An ondispose() callback for use with + sqlite3.StructBinder-created types. */ + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + const fProxy = applyArgcCheck + /** This middle-man proxy is only for use during development, to + confirm that we always pass the proper number of + arguments. We know that the C-level code will always use the + correct argument count. */ + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + }/*installMethod*/; + vh.installMethod.installMethodArgcCheck = false; + + /** + Installs methods into the given StructType-type object. Each + entry in the given methods object must map to a known member of + the given StructType, else an exception will be triggered. + See installMethod() for more details, including the semantics + of the 3rd argument. + + On success, passes its first argument to holdRefence() and + returns this object. Throws on error. + */ + vh.installMethods = function(structType, methods, + applyArgcCheck=vh.installMethod.installMethodArgcCheck){ + for(const k of Object.keys(methods)){ + vh.installMethod(structType, k, methods[k], applyArgcCheck); + } + return vh.holdReference(structType); + }; + + /** + Uses sqlite3_vfs_register() to register the + sqlite3.capi.sqlite3_vfs-type vfs, which must have already been + filled out properly. If the 2nd argument is truthy, the VFS is + registered as the default VFS, else it is not. + + On success, passes its first argument to this.holdReference() and + returns this object. Throws on error. + */ + vh.registerVfs = function(vfs, asDefault=false){ + if(!(vfs instanceof sqlite3.capi.sqlite3_vfs)){ + toss("Expecting a sqlite3_vfs-type argument."); + } + const rc = capi.sqlite3_vfs_register(vfs.pointer, asDefault ? 1 : 0); + if(rc){ + toss("sqlite3_vfs_register(",vfs,") failed with rc",rc); + } + if(vfs.pointer !== capi.sqlite3_vfs_find(vfs.$zName)){ + toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS", + vfs); + } + return vh.holdReference(vfs); + }; + + /** + A wrapper for installMethods() or registerVfs() to reduce + installation of a VFS and/or its I/O methods to a single + call. + + Accepts an object which contains the properties "io" and/or + "vfs", each of which is itself an object with following properties: + + - `struct`: an sqlite3.StructType-type struct. This must be a + populated (except for the methods) object of type + sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the + "vfs" entry). + + - `methods`: an object mapping sqlite3_io_methods method names + (e.g. 'xClose') to JS implementations of those methods. + + For each of those object, this function passes its (`struct`, + `methods`, (optional) `applyArgcCheck`) properties to + this.installMethods(). + + If the `vfs` entry is set, its `struct` property is passed + to this.registerVfs(). The `vfs` entry may optionally have + an `asDefault` property, which gets passed as the 2nd + argument to registerVfs(). + + On success returns this object. Throws on error. + */ + vh.installVfs = function(opt){ + let count = 0; + for(const key of ['io','vfs']){ + const o = opt[key]; + if(o){ + ++count; + this.installMethods(o.struct, o.methods, !!o.applyArgcCheck); + if('vfs'===key) this.registerVfs(o.struct, !!o.asDefault); + } + } + if(!count) toss("Misue: installVfs() options object requires at least", + "one of 'io' or 'vfs' properties."); + return this; + }; + + sqlite3.VfsHelper = vh; +}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 2fc6752694..0c32c1b071 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1867,6 +1867,9 @@ self.sqlite3InitModule = sqlite3InitModule; } importScripts(sqlite3Js); } + self.sqlite3InitModule.__isUnderTest = + true /* disables certain API-internal cleanup so that we can + test internal APIs from here */; self.sqlite3InitModule({ print: log, printErr: error diff --git a/manifest b/manifest index ecfa59693e..a615450fff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Doc\sand\slogging\stext\stweaks\sin\sthe\sOPFS\sasync\sproxy\sand\stest\sapp. -D 2022-11-30T03:08:50.289 +C Refactor\sa\ssignificant\schunk\sof\sthe\sOPFS\ssqlite3_vfs\sinit\scode\sinto\ssqlite3.VfsHelper,\sand\sinternal-use-only\sAPI\sencapsulating\scode\srelevant\sto\screating\snew\sVFSes\sin\sJS.\sIntended\sto\sassist\sin\spending\sexperimentation\swith\san\salternative\sOPFS\sVFS. +D 2022-11-30T05:27:36.071 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,25 +491,26 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile bc1696a1189f4c571b3d878b8f3a67c1f4b52c222f52d356027a5a0c707337c7 +F ext/wasm/GNUmakefile a1a94ff422be90c06263055aec4c569fe3e4ffb3e8cc7f954fd2739aefa52292 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 59e52f579cd3a332d73dae94c91b9579daafb10dd6ada03803f1afa6bdad7689 +F ext/wasm/api/README.md 44d1ea63a990678e373f17ff4b08c2f8599e5a5754b7394e76f7e52164f307c0 +F ext/wasm/api/extern-post-js.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f -F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 +F ext/wasm/api/sqlite3-api-cleanup.js 6a22a3287b34a2b4b0f8127614242e8307cce31ac060eb705a89a6d04dcb1028 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js 2f99c9b2c3c0f53bf40153de20db350378deed825ec0d5cf8f449e9e0081f08b -F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd +F ext/wasm/api/sqlite3-api-opfs.js e36eec4535dab2da186c1fc743b8fb3a37b21a7c61425546d748a3038c25efab +F ext/wasm/api/sqlite3-api-prologue.js 09fa4bf1e29596e599ead649003e01691419ffcb12d0f617152d577741973227 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 315582a4fe3b87ceaff69ad4a67d68a350b8014ac94d6a584d09ea922453c983 +F ext/wasm/api/sqlite3-vfs-helper.js 0b1de47e4407a414db1ee0ca3f9916ae74a9655fe1b1145d4a08e2b52f1ee25f F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa +F ext/wasm/tester1.c-pp.js 5820d578b67835aaf651230663dd207237071ee1f16f11a9c5168500aae04a50 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2064,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 -R 05044488394ac3d7b358270198570ce0 +P 7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b +R 51f96a0acaea8419465c60238f6c368c U stephan -Z cb806f606aa315e69c2e3fceeb852a45 +Z aa663f409b5ddaba03a8fcd513053db4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 12e29fe3c0..4b90be6d7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b \ No newline at end of file +e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 \ No newline at end of file From 36a0163e40eb7097e562dfc6d12af516847de2f3 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 07:17:29 +0000 Subject: [PATCH 70/72] Rename some OPFS JS files. Prevent JS bindings of sqlite3_uri_...() from performing JS-to-C-string argument conversion on their first argument, as doing so is specifically illegal. FossilOrigin-Name: 79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb --- ext/wasm/GNUmakefile | 4 +-- ext/wasm/api/README.md | 12 +++---- ext/wasm/api/sqlite3-api-glue.js | 1 + ext/wasm/api/sqlite3-api-prologue.js | 33 ++++++++++++------- ext/wasm/api/sqlite3-opfs-async-proxy.js | 9 +++-- ext/wasm/api/sqlite3-vfs-helper.js | 28 +++++++++++----- ...qlite3-api-opfs.js => sqlite3-vfs-opfs.js} | 2 +- manifest | 24 +++++++------- manifest.uuid | 2 +- 9 files changed, 68 insertions(+), 47 deletions(-) rename ext/wasm/api/{sqlite3-api-opfs.js => sqlite3-vfs-opfs.js} (99%) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 3c6b3c9a15..eb23dc28bb 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -287,10 +287,10 @@ sqlite3-api.jses += $(dir.common)/whwasmutil.js sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js sqlite3-api.jses += $(sqlite3-api-build-version.js) -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # "External" API files which are part of our distribution diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 37d986584e..3a14229679 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -52,11 +52,6 @@ browser client: Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- **`sqlite3-vfs-helper.js`**\ - This internal-use-only file installs `sqlite3.VfsHelper` for use by - `sqlite3-api-*.js` files which create `sqlite3_vfs` implemenations. - `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after - the library is finished initializing. - **`sqlite3-api-oo1.js`**\ Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other @@ -79,7 +74,12 @@ browser client: a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- **`sqlite3-api-opfs.js`**\ +- **`sqlite3-vfs-helper.js`**\ + This internal-use-only file installs `sqlite3.VfsHelper` for use by + `sqlite3-*.js` files which create `sqlite3_vfs` implemenations. + `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after + the library is finished initializing. +- **`sqlite3-vfs-opfs.js`**\ is an sqlite3 VFS implementation which supports Google Chrome's Origin-Private FileSystem (OPFS) as a storage layer to provide persistent storage for database files in a browser. It requires... diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 86aa1d1813..6c1fcae820 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -69,6 +69,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ const aPtr = wasm.xWrap.argAdapter('*'); wasm.xWrap.argAdapter('sqlite3*', aPtr) + ('sqlite3_filename', aPtr) ('sqlite3_stmt*', aPtr) ('sqlite3_context*', aPtr) ('sqlite3_value*', aPtr) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 6d39581965..59cdab9295 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -913,9 +913,12 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_strlike", "int", "string","string","int"], ["sqlite3_trace_v2", "int", "sqlite3*", "int", "*", "*"], ["sqlite3_total_changes", "int", "sqlite3*"], - ["sqlite3_uri_boolean", "int", "string", "string", "int"], - ["sqlite3_uri_key", "string", "string", "int"], - ["sqlite3_uri_parameter", "string", "string", "string"], + /* Note sqlite3_uri_...() has very specific requirements + for their first C-string arguments, so we cannot perform + any type conversion on those. */ + ["sqlite3_uri_boolean", "int", "sqlite3_filename", "string", "int"], + ["sqlite3_uri_key", "string", "sqlite3_filename", "int"], + ["sqlite3_uri_parameter", "string", "sqlite3_filename", "string"], ["sqlite3_user_data","void*", "sqlite3_context*"], ["sqlite3_value_blob", "*", "sqlite3_value*"], ["sqlite3_value_bytes","int", "sqlite3_value*"], @@ -949,7 +952,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_realloc64", "*","*", "i64"], ["sqlite3_result_int64",undefined, "*", "i64"], ["sqlite3_total_changes64", "i64", ["sqlite3*"]], - ["sqlite3_uri_int64", "i64", ["string", "string", "i64"]], + ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], ["sqlite3_value_int64","i64", "sqlite3_value*"], ]; @@ -1450,9 +1453,6 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( let lip = sqlite3ApiBootstrap.initializersAsync; delete sqlite3ApiBootstrap.initializersAsync; if(!lip || !lip.length) return Promise.resolve(sqlite3); - // Is it okay to resolve these in parallel or do we need them - // to resolve in order? We currently only have 1, so it - // makes no difference. lip = lip.map((f)=>{ const p = (f instanceof Promise) ? f : f(sqlite3); return p.catch((e)=>{ @@ -1460,10 +1460,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( throw e; }); }); - //let p = lip.shift(); - //while(lip.length) p = p.then(lip.shift()); - //return p.then(()=>sqlite3); - return Promise.all(lip).then(()=>{ + const postInit = ()=>{ if(!sqlite3.__isUnderTest){ /* Delete references to internal-only APIs which are used by some initializers. Retain them when running in test mode @@ -1473,7 +1470,19 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( delete sqlite3.StructBinder; } return sqlite3; - }); + }; + if(1){ + /* Run all initializers in sequence. The advantage is that it + allows us to have post-init cleanup defined outside of this + routine at the end of the list and have it run at a + well-defined time. */ + let p = lip.shift(); + while(lip.length) p = p.then(lip.shift()); + return p.then(postInit); + }else{ + /* Run them in an arbitrary order. */ + return Promise.all(lip).then(postInit); + } }, /** scriptInfo ideally gets injected into this object by the diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index da75f139c9..8bf34cc784 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -592,13 +592,12 @@ const installAsyncProxy = function(self){ (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite - does but it leads to immediate contention on journal files. */ + does but it leads to immediate contention on journal files. + Update: this approach reportedly only works for DELETE journal + mode. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ /* sqlite does not lock these files, so go ahead and grab an OPFS - lock. - - https://www.sqlite.org/uri.html - */ + lock. */ fh.xLock = "xOpen"/* Truthy value to keep entry from getting flagged as auto-locked. String value so that we can easily distinguish is later diff --git a/ext/wasm/api/sqlite3-vfs-helper.js b/ext/wasm/api/sqlite3-vfs-helper.js index 9a15dd85fd..f9d3c18c7b 100644 --- a/ext/wasm/api/sqlite3-vfs-helper.js +++ b/ext/wasm/api/sqlite3-vfs-helper.js @@ -183,25 +183,37 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `methods`, (optional) `applyArgcCheck`) properties to this.installMethods(). - If the `vfs` entry is set, its `struct` property is passed - to this.registerVfs(). The `vfs` entry may optionally have - an `asDefault` property, which gets passed as the 2nd - argument to registerVfs(). + If the `vfs` entry is set then: + + - Its `struct` property is passed to this.registerVfs(). The + `vfs` entry may optionally have an `asDefault` property, which + gets passed as the 2nd argument to registerVfs(). + + - If `struct.$zName` is falsy and the entry has a string-type + `name` property, `struct.$zName` is set to the C-string form of + that `name` value before registerVfs() is called. On success returns this object. Throws on error. */ vh.installVfs = function(opt){ let count = 0; - for(const key of ['io','vfs']){ + const propList = ['io','vfs']; + for(const key of propList){ const o = opt[key]; if(o){ ++count; this.installMethods(o.struct, o.methods, !!o.applyArgcCheck); - if('vfs'===key) this.registerVfs(o.struct, !!o.asDefault); + if('vfs'===key){ + if(!o.struct.$zName && 'string'===typeof o.name){ + o.struct.$zName = wasm.allocCString(o.name); + /* Note that we leak that C-string. */ + } + this.registerVfs(o.struct, !!o.asDefault); + } } } - if(!count) toss("Misue: installVfs() options object requires at least", - "one of 'io' or 'vfs' properties."); + if(!count) toss("Misuse: installVfs() options object requires at least", + "one of:", propList); return this; }; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-vfs-opfs.js similarity index 99% rename from ext/wasm/api/sqlite3-api-opfs.js rename to ext/wasm/api/sqlite3-vfs-opfs.js index 5eb3b22b37..7c7eed7efa 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.js @@ -1266,7 +1266,7 @@ const installOpfsVfs = function callee(options){ opfsUtil.rootDirectory = d; log("End of OPFS sqlite3_vfs setup.", opfsVfs); promiseResolve(sqlite3); - }); + }).catch(promiseReject); }else{ promiseResolve(sqlite3); } diff --git a/manifest b/manifest index a615450fff..6bce7f15cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sa\ssignificant\schunk\sof\sthe\sOPFS\ssqlite3_vfs\sinit\scode\sinto\ssqlite3.VfsHelper,\sand\sinternal-use-only\sAPI\sencapsulating\scode\srelevant\sto\screating\snew\sVFSes\sin\sJS.\sIntended\sto\sassist\sin\spending\sexperimentation\swith\san\salternative\sOPFS\sVFS. -D 2022-11-30T05:27:36.071 +C Rename\ssome\sOPFS\sJS\sfiles.\sPrevent\sJS\sbindings\sof\ssqlite3_uri_...()\sfrom\sperforming\sJS-to-C-string\sargument\sconversion\son\stheir\sfirst\sargument,\sas\sdoing\sso\sis\sspecifically\sillegal. +D 2022-11-30T07:17:29.596 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,26 +491,26 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile a1a94ff422be90c06263055aec4c569fe3e4ffb3e8cc7f954fd2739aefa52292 +F ext/wasm/GNUmakefile c6b88ce5f735e29bef2b5da2e095848a7919100521a45e645cbf2456d759d5dd F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 44d1ea63a990678e373f17ff4b08c2f8599e5a5754b7394e76f7e52164f307c0 +F ext/wasm/api/README.md 44a05899d607a792370260a7c9193c9c111a7df06bc3ad1823c92a159526dbf3 F ext/wasm/api/extern-post-js.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 6a22a3287b34a2b4b0f8127614242e8307cce31ac060eb705a89a6d04dcb1028 -F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 +F ext/wasm/api/sqlite3-api-glue.js 03c40b65530d67bb2748b7380aea5fd1534500f812b76a6b401066dcd7fc4116 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js e36eec4535dab2da186c1fc743b8fb3a37b21a7c61425546d748a3038c25efab -F ext/wasm/api/sqlite3-api-prologue.js 09fa4bf1e29596e599ead649003e01691419ffcb12d0f617152d577741973227 +F ext/wasm/api/sqlite3-api-prologue.js e1db3935e1deb1340c1dc0c0e4730b2b88254d616841ebd5bc6bb1b90b32657f F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 315582a4fe3b87ceaff69ad4a67d68a350b8014ac94d6a584d09ea922453c983 -F ext/wasm/api/sqlite3-vfs-helper.js 0b1de47e4407a414db1ee0ca3f9916ae74a9655fe1b1145d4a08e2b52f1ee25f +F ext/wasm/api/sqlite3-opfs-async-proxy.js efb0a7142c64c6a0f4cfbb588e6ea5baac9941364dfb0d40de2a21af1815bed3 +F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 +F ext/wasm/api/sqlite3-vfs-opfs.js cb38965f0f0a52d22bddd3a40312dbdb7826bc90f1a5ce9692b7986cd05bdf43 w ext/wasm/api/sqlite3-api-opfs.js F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b -R 51f96a0acaea8419465c60238f6c368c +P e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 +R e53006d34fb1afaea0f4ce19d105453b U stephan -Z aa663f409b5ddaba03a8fcd513053db4 +Z 3ad9d27606a854f1ad4ff4e5ecee27f6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4b90be6d7e..d65c6da223 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 \ No newline at end of file +79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb \ No newline at end of file From e7f5118bd347e83348417c91eda187d865938e4e Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 07:48:35 +0000 Subject: [PATCH 71/72] OPFS VFS: remove an invalid TODO and fix a property name typo which caused xCheckReservedLock() to always report false. FossilOrigin-Name: 3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 --- ext/wasm/api/sqlite3-vfs-opfs.js | 12 ++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.js b/ext/wasm/api/sqlite3-vfs-opfs.js index 7c7eed7efa..bd258759c4 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.js @@ -659,13 +659,9 @@ const installOpfsVfs = function callee(options){ involve an inherent race condition. For the time being, pending a better solution, we simply report whether the given pFile is open. - - FIXME: we need to ask the async half whether a lock is - held, as it's possible (since long after this method was - implemented) that we do not hold a lock on an OPFS file. */ const f = __openFiles[pFile]; - wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); + wasm.setMemValue(pOut, f.lockType ? 1 : 0, 'i32'); return 0; }, xClose: function(pFile){ @@ -706,7 +702,11 @@ const installOpfsVfs = function callee(options){ mTimeStart('xLock'); const f = __openFiles[pFile]; let rc = 0; - if( capi.SQLITE_LOCK_NONE === f.lockType ) { + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( !f.lockType ) { rc = opRun('xLock', pFile, lockType); if( 0===rc ) f.lockType = lockType; }else{ diff --git a/manifest b/manifest index 6bce7f15cc..8f95e10ebd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\ssome\sOPFS\sJS\sfiles.\sPrevent\sJS\sbindings\sof\ssqlite3_uri_...()\sfrom\sperforming\sJS-to-C-string\sargument\sconversion\son\stheir\sfirst\sargument,\sas\sdoing\sso\sis\sspecifically\sillegal. -D 2022-11-30T07:17:29.596 +C OPFS\sVFS:\sremove\san\sinvalid\sTODO\sand\sfix\sa\sproperty\sname\stypo\swhich\scaused\sxCheckReservedLock()\sto\salways\sreport\sfalse. +D 2022-11-30T07:48:35.434 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1 F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js efb0a7142c64c6a0f4cfbb588e6ea5baac9941364dfb0d40de2a21af1815bed3 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.js cb38965f0f0a52d22bddd3a40312dbdb7826bc90f1a5ce9692b7986cd05bdf43 w ext/wasm/api/sqlite3-api-opfs.js +F ext/wasm/api/sqlite3-vfs-opfs.js 6848abfd6dcc9e2e5ed99193929c735984c1784186d6130839ea5f5bcc7fd7df F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 -R e53006d34fb1afaea0f4ce19d105453b +P 79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb +R d6a46579cca1740ff67e566c55a1e4a7 U stephan -Z 3ad9d27606a854f1ad4ff4e5ecee27f6 +Z c6b01228f796eacb2c5dcc52cd57eff3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d65c6da223..3a518c054a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb \ No newline at end of file +3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 \ No newline at end of file From c5141c9efa564dc6e6d39685b23741d6632dae5a Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 08:37:17 +0000 Subject: [PATCH 72/72] Simplify how the OPFS VFS async proxy copies initial state sent to it from the synchronous side of the connection. Make the lock-wait time a multiple of the wait-loop timeout interval. FossilOrigin-Name: eddafafffa634a42ceeed70aa3fc58be130527612157a4bf4ff9e65c7f6dc26c --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 23 +++-------------------- ext/wasm/api/sqlite3-vfs-opfs.js | 10 ++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 21 insertions(+), 28 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8bf34cc784..d8234d5092 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -298,7 +298,7 @@ const installAsyncProxy = function(self){ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, msBase = 300; + const maxTries = 6, msBase = state.asyncIdleWaitTime * 3; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -316,7 +316,6 @@ const installAsyncProxy = function(self){ } warn("Error getting sync handle for",opName+"(). Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - //await releaseImplicitLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } @@ -835,17 +834,10 @@ const installAsyncProxy = function(self){ o.key = k; o.f = vi; } - /** - waitTime is how long (ms) to wait for each Atomics.wait(). - We need to wake up periodically to give the thread a chance - to do other things. If this is too high (e.g. 500ms) then - even two workers/tabs can easily run into locking errors. - */ - const waitTime = 100; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, waitTime + state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime )){ await releaseImplicitLocks(); continue; @@ -874,20 +866,11 @@ const installAsyncProxy = function(self){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ const opt = data.args; - state.littleEndian = opt.littleEndian; - state.asyncS11nExceptions = opt.asyncS11nExceptions; + for(const k in opt) state[k] = opt[k]; state.verbose = opt.verbose ?? 1; - state.fileBufferSize = opt.fileBufferSize; - state.sabS11nOffset = opt.sabS11nOffset; - state.sabS11nSize = opt.sabS11nSize; - state.sabOP = opt.sabOP; state.sabOPView = new Int32Array(state.sabOP); - state.sabIO = opt.sabIO; state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.opIds = opt.opIds; - state.sq3Codes = opt.sq3Codes; - state.opfsFlags = opt.opfsFlags; Object.keys(vfsAsyncImpls).forEach((k)=>{ if(!Number.isFinite(state.opIds[k])){ toss("Maintenance required: missing state.opIds[",k,"]"); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.js b/ext/wasm/api/sqlite3-vfs-opfs.js index bd258759c4..7b0a0b3228 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.js @@ -268,6 +268,16 @@ const installOpfsVfs = function callee(options){ // Int16Array uses the platform's endianness. return new Int16Array(buffer)[0] === 256; })(); + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = 100; /** Whether the async counterpart should log exceptions to the serialization channel. That produces a great deal of diff --git a/manifest b/manifest index 8f95e10ebd..fe90f1f943 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\sremove\san\sinvalid\sTODO\sand\sfix\sa\sproperty\sname\stypo\swhich\scaused\sxCheckReservedLock()\sto\salways\sreport\sfalse. -D 2022-11-30T07:48:35.434 +C Simplify\show\sthe\sOPFS\sVFS\sasync\sproxy\scopies\sinitial\sstate\ssent\sto\sit\sfrom\sthe\ssynchronous\sside\sof\sthe\sconnection.\sMake\sthe\slock-wait\stime\sa\smultiple\sof\sthe\swait-loop\stimeout\sinterval. +D 2022-11-30T08:37:17.860 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -508,9 +508,9 @@ F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f F ext/wasm/api/sqlite3-api-prologue.js e1db3935e1deb1340c1dc0c0e4730b2b88254d616841ebd5bc6bb1b90b32657f F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js efb0a7142c64c6a0f4cfbb588e6ea5baac9941364dfb0d40de2a21af1815bed3 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.js 6848abfd6dcc9e2e5ed99193929c735984c1784186d6130839ea5f5bcc7fd7df +F ext/wasm/api/sqlite3-vfs-opfs.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb -R d6a46579cca1740ff67e566c55a1e4a7 +P 3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 +R e3e2fc506276773f57deed8b914e8dd1 U stephan -Z c6b01228f796eacb2c5dcc52cd57eff3 +Z 98f64cec6c8eaf42e6c02ddc6003f146 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a518c054a..20dcd52ad4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 \ No newline at end of file +eddafafffa634a42ceeed70aa3fc58be130527612157a4bf4ff9e65c7f6dc26c \ No newline at end of file