Merge fixes from trunk.

FossilOrigin-Name: 1cb65f36c3539302767f551ed53082f054666a7cb2696cef0990ab6747edbc52
This commit is contained in:
drh
2022-10-14 19:56:58 +00:00
8 changed files with 145 additions and 72 deletions
+58 -52
View File
@@ -10,10 +10,10 @@
***********************************************************************
An INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS: a Worker
which manages asynchronous OPFS handles on behalf of a synchronous
API which controls it via a combination of Worker messages,
SharedArrayBuffer, and Atomics.
An Worker which manages asynchronous OPFS handles on behalf of a
synchronous API which controls it via a combination of Worker
messages, SharedArrayBuffer, and Atomics. It is the asynchronous
counterpart of the API defined in sqlite3-api-opfs.js.
Highly indebted to:
@@ -25,13 +25,6 @@
access to the sqlite3 JS/WASM bits, so any bits which it needs (most
notably SQLITE_xxx integer codes) have to be imported into it via an
initialization process.
Potential TODOs:
- When idle for "a long time", close the sync access handle in order
to release the lock, then re-open it on demand. Similarly, delay
fetching of the sync access handle until we need it. The intent
would be to help multi-tab access to a db avoid locking issues.
*/
'use strict';
const toss = function(...args){throw new Error(args.join(' '))};
@@ -42,8 +35,6 @@ if(self.window === self){
toss("This API requires navigator.storage.getDirectory.");
}
//warn("This file is very much experimental and under construction.",self.location.pathname);
/**
Will hold state copied to this object from the syncronous side of
this API.
@@ -111,9 +102,9 @@ const __openFiles = Object.create(null);
/**
Expects an OPFS file path. It gets resolved, such that ".."
components are properly expanded, and returned. If the 2nd
are is true, it's returned as an array of path elements,
else it's returned as an absolute path string.
components are properly expanded, and returned. If the 2nd arg is
true, the result is returned as an array of path elements, else an
absolute path string is returned.
*/
const getResolvedPath = function(filename,splitIt){
const p = new URL(
@@ -124,9 +115,9 @@ const getResolvedPath = function(filename,splitIt){
/**
Takes the absolute path to a filesystem element. Returns an array
of [handleOfContainingDir, filename]. If the 2nd argument is
truthy then each directory element leading to the file is created
along the way. Throws if any creation or resolution fails.
of [handleOfContainingDir, filename]. If the 2nd argument is truthy
then each directory element leading to the file is created along
the way. Throws if any creation or resolution fails.
*/
const getDirForFilename = async function f(absFilename, createDirs = false){
const path = getResolvedPath(absFilename, true);
@@ -161,7 +152,7 @@ const getSyncHandle = async (fh)=>{
try {
//if(1===i) toss("Just testing.");
//TODO? A config option which tells it to throw here
//randomly every now and then.
//randomly every now and then, for testing purposes.
fh.syncHandle = await fh.fileHandle.createSyncAccessHandle();
break;
}catch(e){
@@ -170,7 +161,7 @@ const getSyncHandle = async (fh)=>{
"attempts failed. ",fh.filenameAbs, ":", e.message);
throw e;
}
error("Error getting sync handle. Waiting",ms,
warn("Error getting sync handle. Waiting",ms,
"ms and trying again.",fh.filenameAbs,e);
Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms);
}
@@ -180,6 +171,16 @@ const getSyncHandle = async (fh)=>{
return fh.syncHandle;
};
/**
If the given file-holding object has a sync handle attached to it,
that handle is remove and asynchronously closed. Though it may
sound sensible to continue work as soon as the close() returns
(noting that it's asynchronous), doing so can cause operations
performed soon afterwards, e.g. a call to getSyncHandle() to fail
because they may happen out of order from the close(). OPFS does
not guaranty that the actual order of operations is retained in
such cases. i.e. always "await" on the result of this function.
*/
const closeSyncHandle = async (fh)=>{
if(fh.syncHandle){
log("Closing sync handle for",fh.filenameAbs);
@@ -206,43 +207,50 @@ const affirmNotRO = function(opName,fh){
if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs);
};
const opTimer = Object.create(null);
opTimer.op = undefined;
opTimer.start = undefined;
/**
We track 2 different timers: the "metrics" timer records how much
time we spend performing work. The "wait" timer records how much
time we spend waiting on the underlying OPFS timer. See the calls
to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd()
throughout this file to see how they're used.
*/
const __mTimer = Object.create(null);
__mTimer.op = undefined;
__mTimer.start = undefined;
const mTimeStart = (op)=>{
opTimer.start = performance.now();
opTimer.op = op;
__mTimer.start = performance.now();
__mTimer.op = op;
//metrics[op] || toss("Maintenance required: missing metrics for",op);
++metrics[op].count;
};
const mTimeEnd = ()=>(
metrics[opTimer.op].time += performance.now() - opTimer.start
metrics[__mTimer.op].time += performance.now() - __mTimer.start
);
const waitTimer = Object.create(null);
waitTimer.op = undefined;
waitTimer.start = undefined;
const __wTimer = Object.create(null);
__wTimer.op = undefined;
__wTimer.start = undefined;
const wTimeStart = (op)=>{
waitTimer.start = performance.now();
waitTimer.op = op;
__wTimer.start = performance.now();
__wTimer.op = op;
//metrics[op] || toss("Maintenance required: missing metrics for",op);
};
const wTimeEnd = ()=>(
metrics[waitTimer.op].wait += performance.now() - waitTimer.start
metrics[__wTimer.op].wait += performance.now() - __wTimer.start
);
/**
Set to true by the 'opfs-async-shutdown' command to quite the wait loop.
This is only intended for debugging purposes: we cannot inspect this
file's state while the tight waitLoop() is running.
Gets set to true by the 'opfs-async-shutdown' command to quit the
wait loop. This is only intended for debugging purposes: we cannot
inspect this file's state while the tight waitLoop() is running and
need a way to stop that loop for introspection purposes.
*/
let flagAsyncShutdown = false;
/**
Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
methods. Maintenance reminder: members are in alphabetical order
to simplify finding them.
methods, as well as helpers like mkdir(). Maintenance reminder:
members are in alphabetical order to simplify finding them.
*/
const vfsAsyncImpls = {
'opfs-async-metrics': async ()=>{
@@ -378,11 +386,13 @@ const vfsAsyncImpls = {
const fh = __openFiles[fid];
let rc = 0;
if( !fh.syncHandle ){
wTimeStart('xLock');
try { await getSyncHandle(fh) }
catch(e){
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR;
}
wTimeEnd();
}
storeAndNotify('xLock',rc);
mTimeEnd();
@@ -432,11 +442,11 @@ const vfsAsyncImpls = {
},
xRead: async function(fid,n,offset){
mTimeStart('xRead');
let rc = 0;
let rc = 0, nRead;
const fh = __openFiles[fid];
try{
wTimeStart('xRead');
const nRead = (await getSyncHandle(fh)).read(
nRead = (await getSyncHandle(fh)).read(
fh.sabView.subarray(0, n),
{at: Number(offset)}
);
@@ -446,6 +456,7 @@ const vfsAsyncImpls = {
rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
}
}catch(e){
if(undefined===nRead) wTimeEnd();
error("xRead() failed",e,fh);
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_READ;
@@ -463,9 +474,8 @@ const vfsAsyncImpls = {
await fh.syncHandle.flush();
}catch(e){
state.s11n.storeException(2,e);
}finally{
wTimeEnd();
}
wTimeEnd();
}
storeAndNotify('xSync',rc);
mTimeEnd();
@@ -493,13 +503,13 @@ const vfsAsyncImpls = {
const fh = __openFiles[fid];
if( state.sq3Codes.SQLITE_LOCK_NONE===lockType
&& fh.syncHandle ){
wTimeStart('xUnlock');
try { await closeSyncHandle(fh) }
catch(e){
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR;
/* Maybe we want to not report this? "Destructors do not
throw." */
}
wTimeEnd();
}
storeAndNotify('xUnlock',rc);
mTimeEnd();
@@ -520,9 +530,8 @@ const vfsAsyncImpls = {
error("xWrite():",e,fh);
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_WRITE;
}finally{
wTimeEnd();
}
wTimeEnd();
storeAndNotify('xWrite',rc);
mTimeEnd();
}
@@ -641,7 +650,7 @@ const waitLoop = async function f(){
const o = Object.create(null);
opHandlers[state.opIds[k]] = o;
o.key = k;
o.f = vi || toss("No vfsAsyncImpls[",k,"]");
o.f = vi;
}
/**
waitTime is how long (ms) to wait for each Atomics.wait().
@@ -649,8 +658,6 @@ const waitLoop = async function f(){
to do other things.
*/
const waitTime = 1000;
let lastOpTime = performance.now();
let now;
while(!flagAsyncShutdown){
try {
if('timed-out'===Atomics.wait(
@@ -658,7 +665,6 @@ const waitLoop = async function f(){
)){
continue;
}
lastOpTime = performance.now();
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
@@ -672,7 +678,7 @@ const waitLoop = async function f(){
}catch(e){
error('in waitLoop():',e);
}
};
}
};
navigator.storage.getDirectory().then(function(d){
+11 -11
View File
@@ -1,5 +1,5 @@
C Cancel\sIndexExpr\sobjects\swhen\sthey\sgo\sout\sof\sscope.
D 2022-10-14T02:00:54.507
C Merge\sfixes\sfrom\strunk.
D 2022-10-14T19:56:58.085
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js 40fc1f779e18e08b31404ef832381ffd36b9d11447007aaf96223198324e4467
F ext/wasm/sqlite3-opfs-async-proxy.js 206ce6bbc3c30ad51a37d9c25e3a2712e70b586e0f9a2cf8cb0b9619017c2671
F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8
F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf
F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@@ -570,7 +570,7 @@ F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e
F src/expr.c b60036e4bb9080c741a970df3b3cfffc6292aadec3670b7b04bfae6f662ff245
F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002
F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1
F src/func.c fe2a795ad647ce42054873ac91c43beb7b5d204892903a67f6e7e314379b9d4a
F src/global.c e83ee571b79ee3adc32e380cf554cf1254bc43763d23786c71721fbcdfbbb965
F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19
F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
@@ -579,7 +579,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c b972b1642cd0f7d5ae4964d713b65158a7bfa6a6e8786a9e8e57bd9c0b4a896d
F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b
F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9
F src/main.c b434b8e4aca38419962de3d1c55fb9279807ba6a8802998a427b1635dc8250f6
F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -617,7 +617,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764
F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960
F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c bb18acf4eded647fef88d4d543c673874dbebff516fbeba90a85e6c13f2a58cd
F src/select.c b795f31a326564ffc448bb26f863c64f0796b2a5e0585a3358aff7877b72ed82
F src/shell.c.in 2915eaf22bda89ad6533851a051de4773c249185360fe1fc7b4477b8f9063b2c
F src/sqlite.h.in d9c8a6243fc0a1c270d69db33758e34b810af3462f9bc5b4af113b347e07c69d
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -848,7 +848,7 @@ F test/collate1.test 71a6f27fdc93a92f14d8ab80c05e1937656a5a03197e1a10157314554d6
F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4
F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5
F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd
F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6
F test/collate5.test b1dfeff239ea69ee9225832553f423d37a6184eb730cee06f6846ab4e3c6dbef
F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907
F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868
F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334
@@ -1243,7 +1243,7 @@ F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200
F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7
F test/like.test 5fe0bc37f307aef0a453ce2de4632bdfc0759448f0421c39f6d53caefe905fac
F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da
F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3d3
F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c
F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf
F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e
@@ -2031,8 +2031,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 2e8d4fd4cfd9e82f33c707ba246fe2bb3ca01762cf5ac5905058fbc7adf0abe7
R 6e639b412a6a288ed64157bd3e723cf0
P 0963519371bb7ac02adb323430855760fb9b82a23745e47c504aaf393d22ac34 ed14863dd72e35fa3a23320c3d5a8166515faea39a555c28a27b2d35e701eac4
R a9b714ce8ec9ca5df691ed405bb47472
U drh
Z c5e6e161a1559a577698a45d0fdb903e
Z 5b8c2826f1d75c8b25b3831a35d10fbe
# Remove this line to create a well-formed Fossil manifest.
+1 -1
View File
@@ -1 +1 @@
0963519371bb7ac02adb323430855760fb9b82a23745e47c504aaf393d22ac34
1cb65f36c3539302767f551ed53082f054666a7cb2696cef0990ab6747edbc52
+1 -1
View File
@@ -740,7 +740,7 @@ static int patternCompare(
** c but in the other case and search the input string for either
** c or cx.
*/
if( c<=0x80 ){
if( c<0x80 ){
char zStop[3];
int bMatch;
if( noCase ){
+3 -1
View File
@@ -508,7 +508,9 @@ static const sqlite3_api_routines sqlite3Apis = {
0,
0,
#endif
sqlite3_db_name
sqlite3_db_name,
/* Version 3.40.0 and later */
sqlite3_value_type
};
/* True if x is the directory separator character
+27 -6
View File
@@ -3750,6 +3750,7 @@ typedef struct SubstContext {
int iNewTable; /* New table number */
int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */
ExprList *pEList; /* Replacement expressions */
ExprList *pCList; /* Collation sequences for replacement expr */
} SubstContext;
/* Forward Declarations */
@@ -3791,9 +3792,10 @@ static Expr *substExpr(
#endif
{
Expr *pNew;
Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr;
int iColumn = pExpr->iColumn;
Expr *pCopy = pSubst->pEList->a[iColumn].pExpr;
Expr ifNullRow;
assert( pSubst->pEList!=0 && pExpr->iColumn<pSubst->pEList->nExpr );
assert( pSubst->pEList!=0 && iColumn<pSubst->pEList->nExpr );
assert( pExpr->pRight==0 );
if( sqlite3ExprIsVector(pCopy) ){
sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
@@ -3831,11 +3833,16 @@ static Expr *substExpr(
/* Ensure that the expression now has an implicit collation sequence,
** just as it did when it was a column of a view or sub-query. */
if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){
CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
(pColl ? pColl->zName : "BINARY")
{
CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse,
pSubst->pCList->a[iColumn].pExpr
);
if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){
pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
(pColl ? pColl->zName : "BINARY")
);
}
}
ExprClearProperty(pExpr, EP_Collate);
}
@@ -4028,6 +4035,18 @@ static void renumberCursors(
}
#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
/*
** If pSel is not part of a compound SELECT, return a pointer to its
** expression list. Otherwise, return a pointer to the expression list
** of the leftmost SELECT in the compound.
*/
static ExprList *findLeftmostExprlist(Select *pSel){
while( pSel->pPrior ){
pSel = pSel->pPrior;
}
return pSel->pEList;
}
#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
/*
** This routine attempts to flatten subqueries as a performance optimization.
@@ -4580,6 +4599,7 @@ static int flattenSubquery(
x.iNewTable = iNewParent;
x.isOuterJoin = isOuterJoin;
x.pEList = pSub->pEList;
x.pCList = findLeftmostExprlist(pSub);
substSelect(&x, pParent, 0);
}
@@ -5062,6 +5082,7 @@ static int pushDownWhereTerms(
x.iNewTable = pSrc->iCursor;
x.isOuterJoin = 0;
x.pEList = pSubq->pEList;
x.pCList = findLeftmostExprlist(pSubq);
pNew = substExpr(&x, pNew);
#ifndef SQLITE_OMIT_WINDOWFUNC
if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){
+36
View File
@@ -19,6 +19,8 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix collate5
#
# Tests are organised as follows:
@@ -26,6 +28,7 @@ source $testdir/tester.tcl
# collate5-2.* - Compound SELECT
# collate5-3.* - ORDER BY on compound SELECT
# collate5-4.* - GROUP BY
# collate5-5.* - Collation sequence cases
# Create the collation sequence 'TEXT', purely for asthetic reasons. The
# test cases in this script could just as easily use BINARY.
@@ -289,4 +292,37 @@ do_test collate5-4.3 {
}
} {}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 5.0 {
CREATE TABLE t1(a, b COLLATE nocase);
CREATE TABLE t2(c, d);
INSERT INTO t2 VALUES(1, 'bbb');
}
do_execsql_test 5.1 {
SELECT * FROM (
SELECT a, b FROM t1 UNION ALL SELECT c, d FROM t2
) WHERE b='BbB';
} {1 bbb}
reset_db
do_execsql_test 5.2 {
CREATE TABLE t1(a,b,c COLLATE NOCASE);
INSERT INTO t1 VALUES(NULL,'C','c');
CREATE VIEW v2 AS
SELECT a,b,c FROM t1 INTERSECT SELECT a,b,b FROM t1
WHERE 'eT"3qRkL+oJMJjQ9z0'>=b
ORDER BY a,b,c;
}
do_execsql_test 5.3 {
SELECT * FROM v2;
} { {} C c }
do_execsql_test 5.4 {
SELECT * FROM v2 WHERE c='c';
} { {} C c }
finish_test
+8
View File
@@ -1006,4 +1006,12 @@ do_test like-2.126.3 {
} {126}
do_test like-3.1 {
db eval "SELECT '\u01C0' LIKE '%\x80'"
} {0}
do_test like-3.2 {
db eval "SELECT '\u0080' LIKE '%\x80'"
} {1}
finish_test