From 82bf13796adf380c70badde482e1e05c5d151128 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 26 Jan 2024 20:34:48 +0000
Subject: [PATCH 1/4] Experimental changes that prevent parser stack overflows
by growing the parser stack with heap memory when it reaches its limit.
FossilOrigin-Name: 3fd062905fc20507b7cfc97fa976ac5b57c5b68926bf9136bd5ea4265d2d6528
---
doc/lemon.html | 18 ++++++++
manifest | 21 ++++++----
manifest.uuid | 2 +-
src/parse.y | 4 ++
tool/lemon.c | 23 +++++++++++
tool/lempar.c | 109 ++++++++++++++++++++++++-------------------------
6 files changed, 112 insertions(+), 65 deletions(-)
diff --git a/doc/lemon.html b/doc/lemon.html
index 66665f46f3..4147d9b31e 100644
--- a/doc/lemon.html
+++ b/doc/lemon.html
@@ -683,6 +683,7 @@ other than that, the order of directives in Lemon is arbitrary.
%endif
%extra_argument
%fallback
+%free
%if
%ifdef
%ifndef
@@ -693,6 +694,7 @@ other than that, the order of directives in Lemon is arbitrary.
%parse_accept
%parse_failure
%right
+%realloc
%stack_overflow
%stack_size
%start_symbol
@@ -1200,6 +1202,21 @@ match any input token.
the wildcard token and some other token, the other token is always used.
The wildcard token is only matched if there are no alternatives.
+
+4.4.26 The %realloc and %free directives
+
+The %realloc and %free directives defines function
+that allocate and free heap memory. The signatures of these functions
+should be the same as the realloc() and free() functions from the standard
+C library.
+
+
If both of these functions are defined
+then these functions are used to allocate and free
+memory for supplemental parser stack space, if the initial
+parse stack space is exceeded. The initial parser stack size
+is specified by either %stack_size or the
+-DYYSTACKDEPTH compile-time flag.
+
5.0 Error Processing
@@ -1224,6 +1241,7 @@ to begin parsing a new file. This is what will happen at the very
first syntax error, of course, if there are no instances of the
"error" non-terminal in your grammar.
+
6.0 History of Lemon
diff --git a/manifest b/manifest
index d546c7f2d5..42fb3ce813 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sNEVER()\sto\sa\sbranch\sthat\sis\sno\slonger\sreachable.
-D 2024-01-24T21:08:57.555
+C Experimental\schanges\sthat\sprevent\sparser\sstack\soverflows\sby\sgrowing\sthe\nparser\sstack\swith\sheap\smemory\swhen\sit\sreaches\sits\slimit.
+D 2024-01-26T20:34:48.241
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -40,7 +40,7 @@ F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd
F doc/compile-for-windows.md 50b27d77be96195c66031a3181cb8684ed822327ea834e07f9c014213e5e3bcf
F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f
F doc/jsonb.md 5fab4b8613aa9153fbeb6259297bd4697988af8b3d23900deba588fa7841456b
-F doc/lemon.html 44a53a1d2b42d7751f7b2f478efb23c978e258d794bfd172442307a755b9fa44
+F doc/lemon.html 8b266ff711d2ec7f867c3dca37634963f48a630329908cc282beebfa8c708706
F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
F doc/testrunner.md 8d36ec692cf4994bb66d84a4645b9afa1ce9d47dc12cbf8d437c5a5fb6ddeedb
F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a
@@ -727,7 +727,7 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a
F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd
F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a
-F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec
+F src/parse.y 41926c507955f2c13b10bb344883874cde82ea3ed16cb3b7867a43298d040d79
F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75
F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
@@ -2072,8 +2072,8 @@ F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a
F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5
F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c 19e368bc8e97ff4071115119a7911ca3b0c56eba7926d8ada8b4a86fcc69a176
-F tool/lempar.c 57478ea48420da05faa873c6d1616321caa5464644588c97fbe8e0ea04450748
+F tool/lemon.c 7e5c3c27062c94a40b73a980b8045a4201cb3335165b72ae476040dc513aa533
+F tool/lempar.c fa7ab4dd5bc069ffa276cbd85bea767e6472f4163106b94edd5ad01dd4babdc8
F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669
@@ -2161,8 +2161,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 996cfdf9b5f70408faeaa68ba2ea9494e419be8f2c59d89ab702419056e3569c
-R cc29f59aa57a1c6bea20c14e055ec6a4
+P 9411337a7b3237366768fc708396da53d67a7a17b6cdc5c6f8932c5ab32217a9
+R de3584eaf5d75ac532effc39455b51de
+T *branch * growable-parser-stack
+T *sym-growable-parser-stack *
+T -sym-trunk *
U drh
-Z 22d3dca63255cbf92f3e39d6096754f7
+Z 64ba7e2f532c5b55e706d3959cc36418
# Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 48c5e7a6da..acfd13cd30 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9411337a7b3237366768fc708396da53d67a7a17b6cdc5c6f8932c5ab32217a9
\ No newline at end of file
+3fd062905fc20507b7cfc97fa976ac5b57c5b68926bf9136bd5ea4265d2d6528
\ No newline at end of file
diff --git a/src/parse.y b/src/parse.y
index 19491192e3..0e41f54cf1 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -21,6 +21,10 @@
*/
}
+// Function used to enlarge the parser stack, if needed
+%realloc sqlite3_realloc64
+%free sqlite3_free
+
// All token codes are small integers with #defines that begin with "TK_"
%token_prefix TK_
diff --git a/tool/lemon.c b/tool/lemon.c
index 7804837a06..c578b44bab 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -418,6 +418,8 @@ struct lemon {
char *filename; /* Name of the input file */
char *outname; /* Name of the current output file */
char *tokenprefix; /* A prefix added to token names in the .h file */
+ char *reallocFunc; /* Function to use to allocate stack space */
+ char *freeFunc; /* Function to use to free stack space */
int nconflict; /* Number of parsing conflicts */
int nactiontab; /* Number of entries in the yy_action[] table */
int nlookaheadtab; /* Number of entries in yy_lookahead[] */
@@ -2531,6 +2533,12 @@ static void parseonetoken(struct pstate *psp)
}else if( strcmp(x,"default_type")==0 ){
psp->declargslot = &(psp->gp->vartype);
psp->insertLineMacro = 0;
+ }else if( strcmp(x,"realloc")==0 ){
+ psp->declargslot = &(psp->gp->reallocFunc);
+ psp->insertLineMacro = 0;
+ }else if( strcmp(x,"free")==0 ){
+ psp->declargslot = &(psp->gp->freeFunc);
+ psp->insertLineMacro = 0;
}else if( strcmp(x,"stack_size")==0 ){
psp->declargslot = &(psp->gp->stacksize);
psp->insertLineMacro = 0;
@@ -4501,6 +4509,21 @@ void ReportTable(
fprintf(out,"#define %sARG_FETCH\n",name); lineno++;
fprintf(out,"#define %sARG_STORE\n",name); lineno++;
}
+ if( lemp->reallocFunc ){
+ fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++;
+ }else{
+ fprintf(out,"#define YYREALLOC realloc\n"); lineno++;
+ }
+ if( lemp->freeFunc ){
+ fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++;
+ }else{
+ fprintf(out,"#define YYFREE free\n"); lineno++;
+ }
+ if( lemp->reallocFunc && lemp->freeFunc ){
+ fprintf(out,"#define YYDYNSTACK 1\n"); lineno++;
+ }else{
+ fprintf(out,"#define YYDYNSTACK 0\n"); lineno++;
+ }
if( lemp->ctx && lemp->ctx[0] ){
i = lemonStrlen(lemp->ctx);
while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--;
diff --git a/tool/lempar.c b/tool/lempar.c
index 8cc57897db..d8e64c40dd 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -67,6 +67,9 @@
** ParseARG_STORE Code to store %extra_argument into yypParser
** ParseARG_FETCH Code to extract %extra_argument from yypParser
** ParseCTX_* As ParseARG_ except for %extra_context
+** YYREALLOC Name of the realloc() function to use
+** YYFREE Name of the free() function to use
+** YYDYNSTACK True if stack space should be extended on heap
** YYERRORSYMBOL is the code number of the error symbol. If not
** defined, then do no error processing.
** YYNSTATE the combined number of states.
@@ -101,6 +104,22 @@
# define yytestcase(X)
#endif
+/* Macro to determine if stack space has the ability to grow using
+** heap memory.
+*/
+#if YYSTACKDEPTH<=0 || YYDYNSTACK
+# define YYGROWABLESTACK 1
+#else
+# define YYGROWABLESTACK 0
+#endif
+
+/* Guarantee a minimum number of initial stack slots.
+*/
+#if YYSTACKDEPTH<=0
+# undef YYSTACKDEPTH
+# define YYSTACKDEPTH 2 /* Need a minimum stack size */
+#endif
+
/* Next are the tables used to determine what action to take based on the
** current state and lookahead token. These tables are used to implement
@@ -212,14 +231,9 @@ struct yyParser {
#endif
ParseARG_SDECL /* A place to hold %extra_argument */
ParseCTX_SDECL /* A place to hold %extra_context */
-#if YYSTACKDEPTH<=0
- int yystksz; /* Current side of the stack */
- yyStackEntry *yystack; /* The parser's stack */
- yyStackEntry yystk0; /* First stack entry */
-#else
- yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */
- yyStackEntry *yystackEnd; /* Last entry in the stack */
-#endif
+ yyStackEntry *yystackEnd; /* Last entry in the stack */
+ yyStackEntry *yystack; /* The parser stack */
+ yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */
};
typedef struct yyParser yyParser;
@@ -273,37 +287,45 @@ static const char *const yyRuleName[] = {
#endif /* NDEBUG */
-#if YYSTACKDEPTH<=0
+#if YYGROWABLESTACK
/*
** Try to increase the size of the parser stack. Return the number
** of errors. Return 0 on success.
*/
static int yyGrowStack(yyParser *p){
+ int oldSize = 1 + (int)(p->yystackEnd - p->yystack)/sizeof(p->yystack[0]);
int newSize;
int idx;
yyStackEntry *pNew;
- newSize = p->yystksz*2 + 100;
+ newSize = oldSize*2 + 100;
idx = p->yytos ? (int)(p->yytos - p->yystack) : 0;
- if( p->yystack==&p->yystk0 ){
- pNew = malloc(newSize*sizeof(pNew[0]));
- if( pNew ) pNew[0] = p->yystk0;
+ if( p->yystack==p->yystk0 ){
+ pNew = YYREALLOC(0, newSize*sizeof(pNew[0]));
+ if( pNew==0 ) return 1;
+ memcpy(pNew, p->yystk0, sizeof(p->yystk0));
}else{
- pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+ pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0]));
+ if( pNew==0 ) return 1;
}
- if( pNew ){
- p->yystack = pNew;
- p->yytos = &p->yystack[idx];
+ p->yystack = pNew;
+ p->yytos = &p->yystack[idx];
#ifndef NDEBUG
- if( yyTraceFILE ){
- fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
- yyTracePrompt, p->yystksz, newSize);
- }
-#endif
- p->yystksz = newSize;
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
+ yyTracePrompt, oldSize, newSize);
}
- return pNew==0;
+#endif
+ p->yystackEnd = &p->yystack[newSize-1];
+ return 0;
}
+#endif /* YYGROWABLESTACK */
+
+#if !YYGROWABLESTACK
+/* For builds that do no have a growable stack, yyGrowStack always
+** returns an error.
+*/
+# define yyGrowStack(X) 1
#endif
/* Datatype of the argument to the memory allocated passed as the
@@ -323,24 +345,14 @@ void ParseInit(void *yypRawParser ParseCTX_PDECL){
#ifdef YYTRACKMAXSTACKDEPTH
yypParser->yyhwm = 0;
#endif
-#if YYSTACKDEPTH<=0
- yypParser->yytos = NULL;
- yypParser->yystack = NULL;
- yypParser->yystksz = 0;
- if( yyGrowStack(yypParser) ){
- yypParser->yystack = &yypParser->yystk0;
- yypParser->yystksz = 1;
- }
-#endif
+ yypParser->yystack = yypParser->yystk0;
+ yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
#ifndef YYNOERRORRECOVERY
yypParser->yyerrcnt = -1;
#endif
yypParser->yytos = yypParser->yystack;
yypParser->yystack[0].stateno = 0;
yypParser->yystack[0].major = 0;
-#if YYSTACKDEPTH>0
- yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
-#endif
}
#ifndef Parse_ENGINEALWAYSONSTACK
@@ -427,8 +439,8 @@ static void yy_pop_parser_stack(yyParser *pParser){
void ParseFinalize(void *p){
yyParser *pParser = (yyParser*)p;
while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
-#if YYSTACKDEPTH<=0
- if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack);
+#if YYGROWABLESTACK
+ if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack);
#endif
}
@@ -654,25 +666,19 @@ static void yy_shift(
assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
}
#endif
-#if YYSTACKDEPTH>0
- if( yypParser->yytos>yypParser->yystackEnd ){
- yypParser->yytos--;
- yyStackOverflow(yypParser);
- return;
- }
-#else
- if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){
+ yytos = yypParser->yytos;
+ if( yytos>yypParser->yystackEnd ){
if( yyGrowStack(yypParser) ){
yypParser->yytos--;
yyStackOverflow(yypParser);
return;
}
+ yytos = yypParser->yytos;
+ assert( yytos <= yypParser->yystackEnd );
}
-#endif
if( yyNewState > YY_MAX_SHIFT ){
yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
}
- yytos = yypParser->yytos;
yytos->stateno = yyNewState;
yytos->major = yyMajor;
yytos->minor.yy0 = yyMinor;
@@ -911,19 +917,12 @@ void Parse(
(int)(yypParser->yytos - yypParser->yystack));
}
#endif
-#if YYSTACKDEPTH>0
if( yypParser->yytos>=yypParser->yystackEnd ){
- yyStackOverflow(yypParser);
- break;
- }
-#else
- if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
if( yyGrowStack(yypParser) ){
yyStackOverflow(yypParser);
break;
}
}
-#endif
}
yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor ParseCTX_PARAM);
}else if( yyact <= YY_MAX_SHIFTREDUCE ){
From 51f652de10fa5fdc28b990ec4ed60ff54233750b Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 26 Jan 2024 20:57:19 +0000
Subject: [PATCH 2/4] Bug fixes in the function that expands the parser stack.
FossilOrigin-Name: e91179fe849760771c3508b1e7d75325183e5c3b029752d0a97dbdbd57188b97
---
manifest | 17 +++++++----------
manifest.uuid | 2 +-
test/misc5.test | 4 ++--
tool/lempar.c | 4 ++--
4 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/manifest b/manifest
index 42fb3ce813..e783626ab4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Experimental\schanges\sthat\sprevent\sparser\sstack\soverflows\sby\sgrowing\sthe\nparser\sstack\swith\sheap\smemory\swhen\sit\sreaches\sits\slimit.
-D 2024-01-26T20:34:48.241
+C Bug\sfixes\sin\sthe\sfunction\sthat\sexpands\sthe\sparser\sstack.
+D 2024-01-26T20:57:19.823
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -1413,7 +1413,7 @@ F test/misc1.test 8d138a4926ab90617c1aa29ce26e7785ae2b83a4d3a195d543b7374e05589d
F test/misc2.test 71e746af479119386ac2ed7ab7d81d99970e75b49ffd3e8efffee100b4b5f350
F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e
-F test/misc5.test c4aeaa0fa28faa08f2485309c38db4719e6cd1364215d5687a5b96d340a3fa58
+F test/misc5.test 48de5f4220ff1a27922b14b8cf1b977f73c5a49f82b2ccd66bd86c0e53c447d4
F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
F test/misc7.test d912f3d45c2989191b797504a220ca225d6be80b21acad22ba0d35f4a9ee4579
F test/misc8.test 4db9f8be59834cea08c87e9658014080efa02678ef54a088f84fa5647e81fee0
@@ -2073,7 +2073,7 @@ F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5
F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
F tool/lemon.c 7e5c3c27062c94a40b73a980b8045a4201cb3335165b72ae476040dc513aa533
-F tool/lempar.c fa7ab4dd5bc069ffa276cbd85bea767e6472f4163106b94edd5ad01dd4babdc8
+F tool/lempar.c e560057d6e0f578be64894e70e3d86beee6ae26f4aa5a6fb74fdfff8e75ccb35
F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669
@@ -2161,11 +2161,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9411337a7b3237366768fc708396da53d67a7a17b6cdc5c6f8932c5ab32217a9
-R de3584eaf5d75ac532effc39455b51de
-T *branch * growable-parser-stack
-T *sym-growable-parser-stack *
-T -sym-trunk *
+P 3fd062905fc20507b7cfc97fa976ac5b57c5b68926bf9136bd5ea4265d2d6528
+R 201c8ebb7c943c92d1f840025296ad42
U drh
-Z 64ba7e2f532c5b55e706d3959cc36418
+Z 19d016796a5696ebb7583ab93b83b409
# Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index acfd13cd30..d41ff74d95 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3fd062905fc20507b7cfc97fa976ac5b57c5b68926bf9136bd5ea4265d2d6528
\ No newline at end of file
+e91179fe849760771c3508b1e7d75325183e5c3b029752d0a97dbdbd57188b97
\ No newline at end of file
diff --git a/test/misc5.test b/test/misc5.test
index f7c6048d97..0307c3bb07 100644
--- a/test/misc5.test
+++ b/test/misc5.test
@@ -581,9 +581,9 @@ do_test misc5-7.1 {
append sql "(1+"
append tail ")"
}
- append sql 2$tail
+ append sql 2$tail)
catchsql $sql
-} {1 {parser stack overflow}}
+} {0 {}}
# Parser stack overflow is silently ignored when it occurs while parsing the
# schema and PRAGMA writable_schema is turned on.
diff --git a/tool/lempar.c b/tool/lempar.c
index d8e64c40dd..1ea66e8402 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -293,7 +293,7 @@ static const char *const yyRuleName[] = {
** of errors. Return 0 on success.
*/
static int yyGrowStack(yyParser *p){
- int oldSize = 1 + (int)(p->yystackEnd - p->yystack)/sizeof(p->yystack[0]);
+ int oldSize = 1 + (int)(p->yystackEnd - p->yystack);
int newSize;
int idx;
yyStackEntry *pNew;
@@ -303,7 +303,7 @@ static int yyGrowStack(yyParser *p){
if( p->yystack==p->yystk0 ){
pNew = YYREALLOC(0, newSize*sizeof(pNew[0]));
if( pNew==0 ) return 1;
- memcpy(pNew, p->yystk0, sizeof(p->yystk0));
+ memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0]));
}else{
pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0]));
if( pNew==0 ) return 1;
From 3467698000de73695f7eee8473a16d0981fdd5a6 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 27 Jan 2024 01:35:07 +0000
Subject: [PATCH 3/4] Omit the "parser stack overflow" error message, since it
no longer occurs. Replace it with an ordinary "out of memory".
FossilOrigin-Name: 33f17ec24880937892d47a5a321c43785ca18e03dde28a9a0d50ae86edb29679
---
manifest | 12 ++++++------
manifest.uuid | 2 +-
src/parse.y | 2 +-
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/manifest b/manifest
index e783626ab4..d83d65b92d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Bug\sfixes\sin\sthe\sfunction\sthat\sexpands\sthe\sparser\sstack.
-D 2024-01-26T20:57:19.823
+C Omit\sthe\s"parser\sstack\soverflow"\serror\smessage,\ssince\sit\sno\slonger\soccurs.\nReplace\sit\swith\san\sordinary\s"out\sof\smemory".
+D 2024-01-27T01:35:07.422
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -727,7 +727,7 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a
F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd
F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a
-F src/parse.y 41926c507955f2c13b10bb344883874cde82ea3ed16cb3b7867a43298d040d79
+F src/parse.y 7505b34ab81f17c5af10cc2c4c40e94ece3035ef151fe544e21b9a3b0e4e8531
F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75
F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
@@ -2161,8 +2161,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3fd062905fc20507b7cfc97fa976ac5b57c5b68926bf9136bd5ea4265d2d6528
-R 201c8ebb7c943c92d1f840025296ad42
+P e91179fe849760771c3508b1e7d75325183e5c3b029752d0a97dbdbd57188b97
+R a326e9a9ce6869e2191b567c898735a6
U drh
-Z 19d016796a5696ebb7583ab93b83b409
+Z 3729898b234031ff25c57e8984e9d7de
# Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d41ff74d95..7ea834c5c4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e91179fe849760771c3508b1e7d75325183e5c3b029752d0a97dbdbd57188b97
\ No newline at end of file
+33f17ec24880937892d47a5a321c43785ca18e03dde28a9a0d50ae86edb29679
\ No newline at end of file
diff --git a/src/parse.y b/src/parse.y
index 0e41f54cf1..2d5dc8ee38 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -49,7 +49,7 @@
}
}
%stack_overflow {
- sqlite3ErrorMsg(pParse, "parser stack overflow");
+ sqlite3OomFault(pParse->db);
}
// The name of the generated procedure that implements the parser
From 7659ce22c50d526fcd4c115b79ac6ff667ee70f0 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 27 Jan 2024 02:21:25 +0000
Subject: [PATCH 4/4] Optimizations to ParseFinalize() to make up for the extra
cleanup associated with the allocated parser stack. This branch now runs
faster than trunk and is less than 300 bytes larger.
FossilOrigin-Name: f7290db63cc2568090c14dffc4ea4eadfacb5b94b50a1852ef6eefd9e2e32533
---
manifest | 14 +++++++-------
manifest.uuid | 2 +-
tool/lemon.c | 18 +++++++++++++++++-
tool/lempar.c | 21 ++++++++++++++++++++-
4 files changed, 45 insertions(+), 10 deletions(-)
diff --git a/manifest b/manifest
index d83d65b92d..a87dfc1f4e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Omit\sthe\s"parser\sstack\soverflow"\serror\smessage,\ssince\sit\sno\slonger\soccurs.\nReplace\sit\swith\san\sordinary\s"out\sof\smemory".
-D 2024-01-27T01:35:07.422
+C Optimizations\sto\sParseFinalize()\sto\smake\sup\sfor\sthe\sextra\scleanup\sassociated\nwith\sthe\sallocated\sparser\sstack.\s\sThis\sbranch\snow\sruns\sfaster\sthan\strunk\nand\sis\sless\sthan\s300\sbytes\slarger.
+D 2024-01-27T02:21:25.832
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -2072,8 +2072,8 @@ F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a
F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5
F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c 7e5c3c27062c94a40b73a980b8045a4201cb3335165b72ae476040dc513aa533
-F tool/lempar.c e560057d6e0f578be64894e70e3d86beee6ae26f4aa5a6fb74fdfff8e75ccb35
+F tool/lemon.c db6c77d899f565ab42217d924e5daee5b267724433c7747fb6d3329b3c919fa1
+F tool/lempar.c c85839df516cf3cee74829df59e968646275d327763c64c9e776bac92671f0ac
F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669
@@ -2161,8 +2161,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e91179fe849760771c3508b1e7d75325183e5c3b029752d0a97dbdbd57188b97
-R a326e9a9ce6869e2191b567c898735a6
+P 33f17ec24880937892d47a5a321c43785ca18e03dde28a9a0d50ae86edb29679
+R 20cc75bdc335ec49c6e2f7e4614d84da
U drh
-Z 3729898b234031ff25c57e8984e9d7de
+Z 297b67eb6c1efaf54d4488f8dcd81ddc
# Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 7ea834c5c4..6f41fa279d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-33f17ec24880937892d47a5a321c43785ca18e03dde28a9a0d50ae86edb29679
\ No newline at end of file
+f7290db63cc2568090c14dffc4ea4eadfacb5b94b50a1852ef6eefd9e2e32533
\ No newline at end of file
diff --git a/tool/lemon.c b/tool/lemon.c
index c578b44bab..5b1702f4ee 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -4317,7 +4317,7 @@ void ReportTable(
struct action *ap;
struct rule *rp;
struct acttab *pActtab;
- int i, j, n, sz;
+ int i, j, n, sz, mn, mx;
int nLookAhead;
int szActionType; /* sizeof(YYACTIONTYPE) */
int szCodeType; /* sizeof(YYCODETYPE) */
@@ -4647,6 +4647,22 @@ void ReportTable(
fprintf(out,"#define YY_MIN_REDUCE %d\n", lemp->minReduce); lineno++;
i = lemp->minReduce + lemp->nrule;
fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++;
+
+ /* Minimum and maximum token values that have a destructor */
+ mn = mx = 0;
+ for(i=0; insymbol; i++){
+ struct symbol *sp = lemp->symbols[i];
+
+ if( sp && sp->type!=TERMINAL && sp->destructor ){
+ if( mn==0 || sp->indexindex;
+ if( sp->index>mx ) mx = sp->index;
+ }
+ }
+ if( lemp->tokendest ) mn = 0;
+ if( lemp->vardest ) mx = lemp->nsymbol-1;
+ fprintf(out,"#define YY_MIN_DSTRCTR %d\n", mn); lineno++;
+ fprintf(out,"#define YY_MAX_DSTRCTR %d\n", mx); lineno++;
+
tplt_xfer(lemp->name,in,out,&lineno);
/* Now output the action table and its associates:
diff --git a/tool/lempar.c b/tool/lempar.c
index 1ea66e8402..9be2d409ed 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -83,6 +83,8 @@
** YY_NO_ACTION The yy_action[] code for no-op
** YY_MIN_REDUCE Minimum value for reduce actions
** YY_MAX_REDUCE Maximum value for reduce actions
+** YY_MIN_DSTRCTR Minimum symbol value that has a destructor
+** YY_MAX_DSTRCTR Maximum symbol value that has a destructor
*/
#ifndef INTERFACE
# define INTERFACE 1
@@ -438,7 +440,24 @@ static void yy_pop_parser_stack(yyParser *pParser){
*/
void ParseFinalize(void *p){
yyParser *pParser = (yyParser*)p;
- while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
+
+ /* In-lined version of calling yy_pop_parser_stack() for each
+ ** element left in the stack */
+ yyStackEntry *yytos = pParser->yytos;
+ while( yytos>pParser->yystack ){
+#ifndef NDEBUG
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sPopping %s\n",
+ yyTracePrompt,
+ yyTokenName[yytos->major]);
+ }
+#endif
+ if( yytos->major>=YY_MIN_DSTRCTR && yytos->major<=YY_MAX_DSTRCTR ){
+ yy_destructor(pParser, yytos->major, &yytos->minor);
+ }
+ yytos--;
+ }
+
#if YYGROWABLESTACK
if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack);
#endif