build: add reproducible lookahead auditing

.github/workflows/ci.yml

@@ -45,6 +45,28 @@ jobs:
       - name: Check for grammar ambiguities
         run: ./gradlew checkGrammarAmbiguity --no-daemon
 
+      - name: Restore and save lookahead facts
+        uses: actions/cache@v5
+        with:
+          path: build/lookahead-cache/
+          key: lookahead-facts-${{ runner.os }}-${{ hashFiles('src/test/java/net/sf/jsqlparser/parser/audit/**', 'build.gradle', 'gradle/wrapper/gradle-wrapper.properties') }}-${{ github.sha }}
+          restore-keys: |
+            lookahead-facts-${{ runner.os }}-${{ hashFiles('src/test/java/net/sf/jsqlparser/parser/audit/**', 'build.gradle', 'gradle/wrapper/gradle-wrapper.properties') }}-
+
+      - name: Report changed lookahead contexts
+        env:
+          LOOKAHEAD_BASE: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }}
+        run: ./gradlew lookaheadAudit --no-daemon -PlookaheadBase="$LOOKAHEAD_BASE"
+
+      - name: Upload lookahead review report
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: lookahead-review
+          path: build/reports/lookahead/
+          retention-days: 7
+          if-no-files-found: ignore
+
       - name: Upload grammar warning report
         if: failure()
         uses: actions/upload-artifact@v5
@@ -216,4 +238,4 @@ jobs:
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v5
+        uses: actions/deploy-pages@v5

build.gradle

@@ -265,7 +265,7 @@ tasks.register('xmldoc', Javadoc) {
 }
 
 test {
-    environment = [ 'EXPORT_TEST_TO_FILE': 'False' ]
+    environment 'EXPORT_TEST_TO_FILE', 'False'
     useJUnitPlatform()
 
     // set heap size for the test JVM(s)
@@ -846,3 +846,24 @@ tasks.register('checkGrammarAmbiguity') {
 tasks.named('check') {
     dependsOn('checkGrammarAmbiguity')
 }
+
+// Report base/working-tree context changes without making historical lookahead debt a new gate.
+// The analyzer lives in test sources: no audit classes or dependencies enter the library JAR.
+tasks.register('lookaheadAudit', JavaExec) {
+    group = 'verification'
+    description = 'Reports changed lookahead decisions and stale evidence against a Git base.'
+    dependsOn(testClasses)
+    classpath = sourceSets.test.runtimeClasspath
+    mainClass = 'net.sf.jsqlparser.parser.audit.LookaheadAudit'
+    javaLauncher = javaToolchains.launcherFor(java.toolchain)
+    jvmArgs '-Xss4m', '-Xmx1g', '-Dfile.encoding=UTF-8'
+    workingDir = projectDir
+    doFirst {
+        args = [projectDir.absolutePath,
+                providers.gradleProperty('lookaheadBase').getOrElse('HEAD'),
+                layout.buildDirectory.dir('reports/lookahead').get().asFile.absolutePath,
+                file('config/lookahead-evidence.tsv').absolutePath,
+                providers.gradleProperty('lookaheadFailOnFindings').getOrElse('false'),
+                providers.gradleProperty('lookaheadFull').getOrElse('false')]
+    }
+}

config/lookahead-evidence.tsv

@@ -0,0 +1,3 @@
+# key<TAB>fingerprint<TAB>reason<TAB>regression test[<TAB>context|token-probe]
+ConnectionOrCloudConnectionDefinition	eb5720afb73b771120122d76c643490a1d23e8751f3d516026d96a2e807e0f2a	AT conn and AT CLOUD share the first token; retain the second-token decision.	LookaheadCleanupTest.preservesCloudAndOrdinaryConnections
+LockStatement	e95699a7c5c185663cd0606f88a5613ace1e51f6e2e0d61d2a18ac148b16426d	Bound the complete TABLE token probe to one token while preserving its restriction; this certificate does not cover the complete lock AST.	LookaheadTokenProbeTest and LookaheadCleanupTest.boundedTableKeywordKeepsLockTargetAndWait	token-probe

src/site/sphinx/contribution.rst

@@ -120,6 +120,37 @@ When adding a new keyword token to the Grammar:
     2) If the keyword must be reserved (e. |_| g. core SQL syntax like ``SELECT``, ``FROM``, ``WHERE``), add it to the ``Reserved SQL Keywords`` TOKEN block **after** the ``MAX_NON_RESERVED_WORD`` sentinel.
     3) Verify that existing tests pass and that the keyword can be used as a ``Schema``, ``Table``, ``Column``, ``Function`` or ``Alias`` name where expected.
 
+Review Lookahead Changes
+------------------------------
+
+Run the reusable lookahead report against the commit used as the review baseline:
+
+.. code-block:: shell
+
+    ./gradlew lookaheadAudit -PlookaheadBase=HEAD
+
+The default baseline is ``HEAD``, so this command also examines uncommitted grammar changes. CI uses the pull request base commit. Results are written to ``build/reports/lookahead/report.json``. The analyzer runs with the project's existing JDK and JavaCC dependencies; it does not require Python or modify sources.
+
+The report includes ``inventory.tsv`` and ``inventory.json`` for every explicit decision, plus ``report.json`` for changed contexts, dependency roots and stale evidence. Structural anchors ignore source line numbers and unrelated lookahead insertions. Identical anchors remain ambiguous and cannot reuse an individual evidence record. Triage is a review aid, not permission to remove a guard.
+
+Facts are cached by the complete grammar source and analyzer/generator identity under ``build/lookahead-cache``. A content hit in a separate JVM runs neither JJTree nor JavaCC parsing nor FIRST/FOLLOW iteration. External Java/resource/build fingerprints are rebound on every invocation, including cache hits. CI restores and saves this directory separately from Gradle's dependency cache. Invalid, truncated or incompatible cache files cause a rebuild. These are local build artifacts, not an authorization or a semantic proof.
+
+A changed grammar still runs JJTree and a complete JavaCC parse and semantic check. The analyzer then recomputes changed FIRST components, call-site FOLLOW equations and affected decision rows; unchanged inputs reuse their facts. Components restart from bottom when invalidated, so deletions and shrinking sets cannot retain old tokens. Actual calls and calls inside probes have separate roles: runtime FOLLOW uses actual calls, while probe facts and symbolic continuation dependencies include probe calls. The ``analysisBefore`` and ``analysisCurrent`` statistics show front-end runs, recomputation and reuse. Use ``-PlookaheadFull=true`` to bypass this custom cache and incremental reuse for a cold comparison; Gradle's ``--no-build-cache`` alone does not disable it.
+
+Full context evidence includes actual callers, their complete callee closures (including probes), and symbolic continuations beyond FOLLOW1. Caller prefixes and competing alternatives can restrict which inputs reach a callee, so changes in a caller's other branches can conservatively invalidate evidence too. Reuse independent of the whole grammar is limited to an intentionally narrow, effect-free grammar subset. Unknown Java, assignments, signatures, declarations, predicates, recovery, lexer actions, lexical states/transitions and custom runtime hooks retain whole-grammar fallback. JSqlParser's production grammar contains opaque Java, so an unrelated production edit can invalidate full context evidence even when its FIRST/FOLLOW rows are reused.
+
+``config/lookahead-evidence.tsv`` has four columns: key, fingerprint, reason and regression reference. A fifth column explicitly selects ``context`` (the default) or ``token-probe``. Existing four-column records keep their full context meaning. A token certificate is available only for one complete, positive, unnested single-token probe or token set with no Java side effects or predicate. It binds the token decision, complete lexer/options, generator/analyzer identity and external inputs. It can survive an unrelated production edit, but certifies only that local token restriction; it cannot stand in for AST, continuation, error handling or complete cleanup validation. Unknown scope names and unsupported token certificates fail analysis.
+
+Production Java/resources, Maven configuration under ``.mvn/``, Gradle configuration under ``gradle/``, root build files, wrappers, the dedicated cleanup regression and analyzer sources are watched. Full token definitions, including named inline regexes, are bound. Unrecognized watched symlinks fail analysis. Commit-derived version metadata is excluded. Evidence covers tracked static inputs and tested behavior, not every possible runtime configuration.
+
+The report writes current unambiguous fingerprints to ``binding-candidates.tsv``. Review the scope and rerun the referenced tests before updating a stale record. A production name is accepted only when it identifies one decision; otherwise use a unique structural anchor or disambiguate the case. Never copy a narrower fingerprint into a full-context record to silence invalidation.
+
+Complete one-token syntactic probes can be bounded as ``LOOKAHEAD(1, token)``. Keep the explicit restriction: deleting the probe can widen a called production's FIRST set, and nested lookaheads have separate generator rules. Non-nullable branches with disjoint FIRST sets can suggest default dispatch, but imperative JavaCC code still needs review. Retain state/dialect/interruption checks, verify complete consumption and AST behavior, and test malformed neighbors. This change bounds 57 complete token probes and removes one investigated redundant depth-three dispatch in ``DBMSDestination``; it does not remove every historical lookahead.
+
+Error locations can change when a rejected branch is entered directly. In ``DBMSDestination``, ``EXA AT conn TABLE`` remains invalid, but now reports the missing table name after consuming ``TABLE`` instead of falling through to the ``STATEMENT`` alternative. The dedicated regression checks this consumption boundary; exact diagnostic text is not promised unchanged.
+
+The report does not make historical debt a mandatory gate. ``-PlookaheadFailOnFindings=true`` opts into failure when contexts need review; generation or analysis errors always fail. Ordinary ``check`` runs cache, cold/incremental oracle, dependency, token-probe and parser regressions. Full applicable checks and paired project JMH measurements are still required before publishing a cleanup.
+
 
 Commit a Pull Request
 ---------------------------------

src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java

@@ -0,0 +1,163 @@
+/*-
+ * #%L
+ * JSQLParser library
+ * %%
+ * Copyright (C) 2004 - 2019 JSQLParser
+ * %%
+ * Dual licensed under GNU LGPL 2.1 or Apache License 2.0
+ * #L%
+ */
+package net.sf.jsqlparser.parser;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed;
+import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect;
+import net.sf.jsqlparser.statement.ConnectionDefinition;
+import net.sf.jsqlparser.statement.CloudConnectionDefinition;
+import net.sf.jsqlparser.statement.export.DBMSDestination;
+import org.junit.jupiter.api.Test;
+import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.select.Select;
+import net.sf.jsqlparser.statement.select.MySqlSelectIntoClause;
+import net.sf.jsqlparser.statement.execute.Execute;
+import net.sf.jsqlparser.statement.lock.LockStatement;
+import net.sf.jsqlparser.statement.create.rule.CreateRule;
+import net.sf.jsqlparser.statement.insert.Insert;
+import net.sf.jsqlparser.statement.notify.NotifyStatement;
+import net.sf.jsqlparser.expression.LongValue;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class LookaheadCleanupTest {
+    @ParameterizedTest
+    @ValueSource(strings = {"EXA AT conn TABLE t", "ORA AT conn TABLE schemaName.tableName",
+            "JDBC DRIVER = 'driverName' AT conn TABLE \"t\"",
+            "EXA AT conn TABLE t (c1, c2) REPLACE TRUNCATE"})
+    void dispatchesTableDestinationsWithCompleteConsumption(String sql) throws Exception {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        DBMSDestination destination = parser.DBMSDestination();
+        assertNotNull(destination.getTable());
+        assertNull(destination.getStatement());
+        assertNotNull(destination.getConnectionDefinition());
+        assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true,
+                p -> p.withDialect(Dialect.EXASOL));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"EXA AT conn STATEMENT 'insert into t values (?)'",
+            "ORA AT conn STATEMENT 'insert into t values (?)'"})
+    void preservesStatementDestinations(String sql) throws Exception {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        DBMSDestination destination = parser.DBMSDestination();
+        assertNull(destination.getTable());
+        assertEquals("insert into t values (?)", destination.getStatement().getValue());
+        assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true,
+                p -> p.withDialect(Dialect.EXASOL));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"EXA AT conn TABLE", "ORA AT conn STATEMENT", "EXA AT conn SELECT t"})
+    void rejectsIncompleteAndWrongDestinationPrefixes(String sql) {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        assertThrows(ParseException.class, parser::DBMSDestination);
+        if (sql.endsWith("TABLE")) {
+            assertEquals(CCJSqlParserConstants.K_TABLE, parser.getToken(0).kind);
+            assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"AT conn", "AT '127.0.0.1'", "AT CLOUD NONE conn",
+            "AT CLOUD AZURE BLOBSTORAGE conn"})
+    void preservesCloudAndOrdinaryConnections(String sql) throws Exception {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        ConnectionDefinition connection = parser.ConnectionOrCloudConnectionDefinition();
+        if (sql.startsWith("AT CLOUD")) {
+            assertInstanceOf(CloudConnectionDefinition.class, connection);
+        } else {
+            assertEquals(ConnectionDefinition.class, connection.getClass());
+        }
+        assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO CSV " + sql + " FILE 'file.csv'",
+                true, p -> p.withDialect(Dialect.EXASOL));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"SELECT id FROM t LIMIT 7 OFFSET 2",
+            "SELECT id FROM t UNION ALL SELECT id FROM u LIMIT 7 OFFSET 2"})
+    void boundedSelectTailsKeepTheirAstAndFollowingStatement(String sql) throws Exception {
+        Select select = (Select) complete(sql);
+        assertEquals(7, ((LongValue) select.getLimit().getRowCount()).getValue());
+        assertEquals(2, ((LongValue) select.getOffset().getOffset()).getValue());
+        assertEquals("SELECT 42", CCJSqlParserUtil.parseStatements(sql + "; SELECT 42").get(1)
+                .toString());
+    }
+
+    @Test
+    void boundedFetchKeepsItsCount() throws Exception {
+        Select select = (Select) complete("SELECT id FROM t OFFSET 2 ROWS FETCH NEXT 7 ROWS ONLY");
+        assertEquals(2, ((LongValue) select.getOffset().getOffset()).getValue());
+        assertEquals(7, ((LongValue) select.getFetch().getExpression()).getValue());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"SELECT a INTO OUTFILE '/tmp/result.txt' FIELDS TERMINATED BY ',' "
+            + "OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY 'end' FROM t",
+            "SELECT a FROM t INTO OUTFILE '/tmp/result.txt' COLUMNS TERMINATED BY ',' "
+                    + "OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY 'end'"})
+    void boundedOutfileOptionsKeepTheirOrderingAndValues(String sql) throws Exception {
+        MySqlSelectIntoClause into = ((Select) complete(sql)).getPlainSelect()
+                .getMySqlSelectIntoClause();
+        assertEquals(MySqlSelectIntoClause.Type.OUTFILE, into.getType());
+        assertEquals("/tmp/result.txt", into.getFileName().getValue());
+        assertEquals(",", into.getFieldsTerminatedBy().getValue());
+        assertTrue(into.isFieldsOptionallyEnclosed());
+        assertEquals("end", into.getLinesTerminatedBy().getValue());
+    }
+
+    @Test
+    void boundedParenthesesKeepExecutionArgumentsAndRuleActions() throws Exception {
+        Execute execute = (Execute) complete("CALL p (:a, :b)");
+        assertEquals("p", execute.getName());
+        assertEquals(2, execute.getExprList().size());
+        assertEquals(":a", execute.getExprList().get(0).toString());
+        assertEquals(":b", execute.getExprList().get(1).toString());
+        CreateRule rule = (CreateRule) complete("CREATE RULE r AS ON INSERT TO t DO "
+                + "(INSERT INTO log VALUES (9); NOTIFY ch, 'changed')");
+        assertEquals(2, rule.getActions().size());
+        assertEquals("log", ((Insert) rule.getActions().get(0)).getTable().getName());
+        assertEquals("changed",
+                ((NotifyStatement) rule.getActions().get(1)).getPayload().getValue());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"LOCK TABLE a IN SHARE MODE WAIT 5", "LOCK a IN SHARE MODE WAIT 5"})
+    void boundedTableKeywordKeepsLockTargetAndWait(String sql) throws Exception {
+        LockStatement lock = (LockStatement) complete(sql);
+        assertEquals(sql.startsWith("LOCK TABLE"), lock.isUseTableKeyword());
+        assertEquals("a", lock.getTable().getName());
+        assertEquals("SHARE", lock.getLockMode().getValue());
+        assertEquals(5L, lock.getWaitSeconds());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"SELECT id FROM t LIMIT", "SELECT id FROM t OFFSET",
+            "SELECT id FROM t FETCH NEXT", "CALL p (", "CALL p (:a,)", "LOCK TABLE",
+            "CREATE RULE r AS ON INSERT TO t DO (NOTIFY ch", "SELECT a FROM t INTO OUTFILE 'x' "
+                    + "LINES TERMINATED BY 'end' FIELDS TERMINATED BY ','"})
+    void boundedGuardsRejectIncompleteAndReorderedNeighbors(String sql) {
+        assertThrows(net.sf.jsqlparser.JSQLParserException.class,
+                () -> CCJSqlParserUtil.parse(sql));
+    }
+
+    private static Statement complete(String sql) throws Exception {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        Statement statement = parser.Statement();
+        assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        assertEquals(statement.toString(), CCJSqlParserUtil.parse(statement.toString()).toString());
+        return statement;
+    }
+
+}