001package gudusoft.gsqlparser.parser;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.TBaseType;
005import gudusoft.gsqlparser.TCustomLexer;
006import gudusoft.gsqlparser.TCustomParser;
007import gudusoft.gsqlparser.TCustomSqlStatement;
008import gudusoft.gsqlparser.TLexerClickhouse;
009import gudusoft.gsqlparser.TParserClickhouse;
010import gudusoft.gsqlparser.TGSqlParser;
011import gudusoft.gsqlparser.TSourceToken;
012import gudusoft.gsqlparser.TSourceTokenList;
013import gudusoft.gsqlparser.TStatementList;
014import gudusoft.gsqlparser.TSyntaxError;
015import gudusoft.gsqlparser.EFindSqlStateType;
016import gudusoft.gsqlparser.ETokenType;
017import gudusoft.gsqlparser.ETokenStatus;
018import gudusoft.gsqlparser.ESqlStatementType;
019import gudusoft.gsqlparser.EErrorType;
020import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
021import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
022import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
023import gudusoft.gsqlparser.compiler.TContext;
024import gudusoft.gsqlparser.sqlenv.TSQLEnv;
025import gudusoft.gsqlparser.compiler.TGlobalScope;
026import gudusoft.gsqlparser.compiler.TFrame;
027
028import java.util.ArrayList;
029import java.util.List;
030import java.util.Stack;
031
032/**
033 * ClickHouse database SQL parser implementation.
034 *
035 * <p>This parser handles ClickHouse-specific SQL syntax including:
036 * <ul>
037 *   <li>Standard SQL DML/DDL (SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, ALTER)</li>
038 *   <li>ClickHouse-specific ENGINE clause in CREATE TABLE</li>
039 *   <li>PREWHERE, FINAL, SAMPLE, ARRAY JOIN clauses</li>
040 *   <li>FORMAT and SETTINGS clauses</li>
041 *   <li>ClickHouse data types (Array, Tuple, Map, Nullable, LowCardinality, etc.)</li>
042 * </ul>
043 *
044 * <p><b>Design Notes:</b>
045 * <ul>
046 *   <li>Extends {@link AbstractSqlParser}</li>
047 *   <li>Based on MySQL grammar (ClickHouse shares backtick quoting, # comments, ENGINE clause)</li>
048 *   <li>No stored procedure support (ClickHouse has no stored procedures)</li>
049 *   <li>Delimiter character: ';' for SQL statements</li>
050 * </ul>
051 *
052 * @see AbstractSqlParser
053 * @see TLexerClickhouse
054 * @see TParserClickhouse
055 * @since 3.2.0.0
056 */
057public class ClickhouseSqlParser extends AbstractSqlParser {
058
059    // ========== Lexer and Parser Instances ==========
060
061    /** The ClickHouse lexer used for tokenization (public for TGSqlParser.getFlexer()) */
062    public TLexerClickhouse flexer;
063    private TParserClickhouse fparser;
064
065    /**
066     * PARALLEL and WITH of each {@code stmt1 PARALLEL WITH stmt2}, as
067     * clickhousePostTokenAdjust leaves them: the splitter ends stmt1 at them
068     * and gives them to neither statement.
069     */
070    private final java.util.Set<TSourceToken> parallelWithSeparators =
071            java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<TSourceToken, Boolean>());
072
073    /**
074     * Tokens of INSERT ... FORMAT rows whose text went into the first token of their rows; the pre-pass
075     * drops them all at its end, so a script of many inserts is not re-shifted once per insert.
076     */
077    private final java.util.Set<TSourceToken> foldedRowTokens =
078            java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<TSourceToken, Boolean>());
079
080    // ========== Constructor ==========
081
082    /**
083     * Construct ClickHouse SQL parser.
084     * <p>
085     * Configures the parser for ClickHouse database with default delimiter: semicolon (;)
086     */
087    public ClickhouseSqlParser() {
088        super(EDbVendor.dbvclickhouse);
089
090        // Set delimiter character - ClickHouse uses semicolon
091        this.delimiterChar = ';';
092        this.defaultDelimiterStr = ";";
093
094        // Create lexer once - will be reused for all parsing operations
095        this.flexer = new TLexerClickhouse();
096        this.flexer.delimiterchar = this.delimiterChar;
097        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;
098
099        // CRITICAL: Set lexer for inherited getanewsourcetoken() method
100        this.lexer = this.flexer;
101
102        // Create parser once - will be reused for all parsing operations
103        this.fparser = new TParserClickhouse(null);
104        this.fparser.lexer = this.flexer;
105    }
106
107    // ========== AbstractSqlParser Abstract Methods Implementation ==========
108
109    @Override
110    protected TCustomLexer getLexer(ParserContext context) {
111        return this.flexer;
112    }
113
114    @Override
115    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
116        this.fparser.sourcetokenlist = tokens;
117        return this.fparser;
118    }
119
120    @Override
121    protected void tokenizeVendorSql() {
122        doclickhousetexttotokenlist();
123    }
124
125    @Override
126    protected void setupVendorParsersForExtraction() {
127        this.fparser.sqlcmds = this.sqlcmds;
128        this.fparser.sourcetokenlist = this.sourcetokenlist;
129    }
130
131    @Override
132    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
133        doclickhousegetrawsqlstatements(builder);
134    }
135
136    @Override
137    protected TStatementList performParsing(ParserContext context,
138                                           TCustomParser parser,
139                                           TCustomParser secondaryParser,
140                                           TSourceTokenList tokens,
141                                           TStatementList rawStatements) {
142        this.sourcetokenlist = tokens;
143        this.parserContext = context;
144        this.sqlstatements = rawStatements;
145
146        // Initialize sqlcmds for the parser
147        this.sqlcmds = SqlCmdsFactory.get(vendor);
148        this.fparser.sqlcmds = this.sqlcmds;
149
150        // Initialize global context for statement parsing
151        initializeGlobalContext();
152
153        // Parse each statement
154        for (int i = 0; i < sqlstatements.size(); i++) {
155            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);
156
157            try {
158                stmt.setFrameStack(frameStack);
159                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());
160
161                // Handle error recovery for CREATE TABLE statements if enabled
162                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
163                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
164                    handleCreateTableErrorRecovery(stmt);
165                }
166
167                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
168                    copyErrorsFromStatement(stmt);
169                }
170            } catch (Exception ex) {
171                handleStatementParsingException(stmt, i, ex);
172                continue;
173            }
174        }
175
176        // Clean up frame stack
177        if (globalFrame != null) {
178            globalFrame.popMeFromStack(frameStack);
179        }
180
181        return this.sqlstatements;
182    }
183
184    /**
185     * Handle error recovery for CREATE TABLE statements.
186     */
187    private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
188        if ((stmt.sqlstatementtype != ESqlStatementType.sstcreatetable) || TBaseType.c_createTableStrictParsing) {
189            return;
190        }
191
192        int nested = 0;
193        boolean isIgnore = false, isFoundIgnoreToken = false;
194        TSourceToken firstIgnoreToken = null;
195
196        for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
197            TSourceToken st = stmt.sourcetokenlist.get(k);
198            if (isIgnore) {
199                if (st.issolidtoken() && (st.tokencode != ';')) {
200                    isFoundIgnoreToken = true;
201                    if (firstIgnoreToken == null) {
202                        firstIgnoreToken = st;
203                    }
204                }
205                if (st.tokencode != ';') {
206                    st.tokencode = TBaseType.sqlpluscmd;
207                }
208                continue;
209            }
210            if (st.tokencode == (int) ')') {
211                nested--;
212                if (nested == 0) {
213                    boolean isSelect = false;
214                    TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
215                    if (st1 != null) {
216                        TSourceToken st2 = st.searchToken((int) '(', 2);
217                        if (st2 != null) {
218                            TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
219                            isSelect = (st3 != null);
220                        }
221                    }
222                    if (!isSelect) isIgnore = true;
223                }
224            } else if (st.tokencode == (int) '(') {
225                nested++;
226            }
227        }
228
229        if (isFoundIgnoreToken) {
230            stmt.clearError();
231            stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree());
232        }
233    }
234
235    // ========== ClickHouse-Specific Tokenization ==========
236
237    /**
238     * Perform ClickHouse-specific tokenization.
239     * <p>
240     * Based on MySQL tokenization but simplified - no DELIMITER command,
241     * no stored procedure handling. Handles ClickHouse-specific token
242     * transformations including :: typecast operator splitting.
243     */
244    private void doclickhousetexttotokenlist() {
245        TSourceToken asourcetoken;
246        int yychar;
247
248        asourcetoken = getanewsourcetoken();
249        if (asourcetoken == null) return;
250        yychar = asourcetoken.tokencode;
251
252        while (yychar > 0) {
253            // Handle :: typecast operator: the lexer produces mysqllabel (identifier:)
254            // followed by bind_v (:name) for "identifier::TypeName". We need to
255            // rewrite as CAST(identifier AS TypeName).
256            if (yychar == TBaseType.mysqllabel) {
257                // Append BEFORE fetching the lookahead: getanewsourcetoken's
258                // whitespace merge reads sourcetokenlist.get(curpos), which is
259                // out of bounds while this token is held back — "SELECT {a: b}"
260                // (label then whitespace) crashed the whole tokenize and the
261                // parser silently returned zero tokens. The bind_v rewrite
262                // removes the token again below.
263                sourcetokenlist.add(asourcetoken);
264                TSourceToken nextToken = getanewsourcetoken();
265                if (nextToken != null && nextToken.tokencode == TBaseType.bind_v) {
266                    sourcetokenlist.remove(sourcetokenlist.size() - 1);
267                    // We have mysqllabel + bind_v pattern = "ident:" + ":TypeName"
268                    // Rewrite as: CAST ( identifier AS TypeName )
269                    String labelText = asourcetoken.toString();
270                    String identText = labelText.substring(0, labelText.length() - 1);
271                    String bindText = nextToken.toString();
272                    String typeText = bindText.substring(1);
273                    int line = (int) asourcetoken.lineNo;
274                    int col = (int) asourcetoken.columnNo;
275
276                    // Check if preceding token is '.' (qualified name like t1.col::Type)
277                    // If so, pull qualified name prefix into the CAST expression
278                    List<TSourceToken> qualifiedPrefix = new ArrayList<>();
279                    int lastIdx = sourcetokenlist.size() - 1;
280                    // Skip trailing whitespace
281                    while (lastIdx >= 0 && (sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttwhitespace
282                            || sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttreturn)) {
283                        lastIdx--;
284                    }
285                    if (lastIdx >= 0 && sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttperiod) {
286                        // Walk back collecting name.name.name. pattern
287                        int prefixEnd = lastIdx;
288                        int idx = lastIdx;
289                        while (idx >= 0) {
290                            TSourceToken t = sourcetokenlist.get(idx);
291                            if (t.tokentype == ETokenType.ttperiod) {
292                                idx--;
293                            } else if (t.tokentype == ETokenType.ttidentifier || t.tokentype == ETokenType.ttkeyword) {
294                                idx--;
295                                // Skip whitespace between identifier and dot
296                                while (idx >= 0 && (sourcetokenlist.get(idx).tokentype == ETokenType.ttwhitespace
297                                        || sourcetokenlist.get(idx).tokentype == ETokenType.ttreturn)) {
298                                    idx--;
299                                }
300                                // Check if next non-ws token is another dot (continue) or stop
301                                if (idx >= 0 && sourcetokenlist.get(idx).tokentype == ETokenType.ttperiod) {
302                                    continue;
303                                } else {
304                                    break;
305                                }
306                            } else {
307                                break;
308                            }
309                        }
310                        int prefixStart = idx + 1;
311                        // Collect prefix tokens and remove from sourcetokenlist
312                        for (int i = prefixStart; i <= prefixEnd; i++) {
313                            qualifiedPrefix.add(sourcetokenlist.get(i));
314                        }
315                        // Remove prefix tokens from end of sourcetokenlist
316                        while (sourcetokenlist.size() > prefixStart) {
317                            sourcetokenlist.remove(sourcetokenlist.size() - 1);
318                        }
319                    }
320
321                    // Emit: CAST (
322                    sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col));
323                    sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col));
324
325                    // Re-emit qualified prefix if any (e.g., "t1.")
326                    for (TSourceToken prefixToken : qualifiedPrefix) {
327                        prefixToken.posinlist = sourcetokenlist.size();
328                        sourcetokenlist.add(prefixToken);
329                    }
330
331                    // Emit: identifier or integer
332                    asourcetoken.setAstext(identText);
333                    if (identText.matches("\\d+")) {
334                        asourcetoken.tokencode = TBaseType.iconst;
335                        asourcetoken.tokentype = ETokenType.ttnumber;
336                    } else {
337                        asourcetoken.tokencode = TBaseType.ident;
338                        asourcetoken.tokentype = ETokenType.ttidentifier;
339                    }
340                    sourcetokenlist.add(asourcetoken);
341
342                    // Emit: AS
343                    sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col));
344
345                    // Emit: TypeName
346                    nextToken.setAstext(typeText);
347                    nextToken.tokencode = TBaseType.ident;
348                    nextToken.tokentype = ETokenType.ttidentifier;
349                    sourcetokenlist.add(nextToken);
350
351                    // Check for parameterized types: TypeName(params)
352                    emitTypeParamsAndClose(line, col);
353
354                    asourcetoken = getanewsourcetoken();
355                    if (asourcetoken == null) break;
356                    yychar = asourcetoken.tokencode;
357                    continue;
358                } else {
359                    // Not followed by bind_v; the mysqllabel is already in the
360                    // list, continue with the lookahead token.
361                    if (nextToken == null) break;
362                    asourcetoken = nextToken;
363                    yychar = asourcetoken.tokencode;
364                    continue;
365                }
366            }
367
368            // Handle standalone :: typecast token (e.g., after string literals: '2024-01-15'::Date)
369            if (yychar == TBaseType.typecast) {
370                rewriteStandaloneTypecast(asourcetoken);
371                asourcetoken = getanewsourcetoken();
372                if (asourcetoken == null) break;
373                yychar = asourcetoken.tokencode;
374                continue;
375            }
376
377            if (yychar == TBaseType.bind_v) {
378                // ClickHouse has no :name bind variables; a bind_v that
379                // survived the :: rewrite above is a map-literal colon glued
380                // to its value ({'key1':1}). Split into ':' plus the value.
381                String bindText = asourcetoken.toString();
382                int bLine = (int) asourcetoken.lineNo;
383                int bCol = (int) asourcetoken.columnNo;
384                sourcetokenlist.add(createSyntheticToken(":", ':', ETokenType.ttcolon, bLine, bCol));
385                String valText = bindText.substring(1);
386                boolean allDigits = valText.length() > 0;
387                for (int ci = 0; ci < valText.length(); ci++) {
388                    if (!Character.isDigit(valText.charAt(ci))) { allDigits = false; break; }
389                }
390                if (allDigits) {
391                    sourcetokenlist.add(createSyntheticToken(valText, TBaseType.iconst, ETokenType.ttnumber, bLine, bCol + 1));
392                } else {
393                    sourcetokenlist.add(createSyntheticToken(valText, TBaseType.ident, ETokenType.ttidentifier, bLine, bCol + 1));
394                }
395                asourcetoken = getanewsourcetoken();
396                if (asourcetoken == null) break;
397                yychar = asourcetoken.tokencode;
398                continue;
399            }
400
401            // Tuple element access a.2: the lexer greedily reads ".2" as a
402            // float constant. When a float starting with '.' directly follows
403            // an identifier or a closing bracket, split it into a dedicated
404            // tuple-access dot token plus an integer constant. The dot gets
405            // its own token (CH_TUPLE_DOT) rather than '.' so the grammar
406            // rule cannot collide with qualified names like left.col.
407            if ((yychar == TBaseType.fconst) && asourcetoken.toString().startsWith(".")
408                    && asourcetoken.toString().indexOf('.', 1) < 0) {
409                TSourceToken lastSolid = null;
410                for (int li = sourcetokenlist.size() - 1; li >= 0; li--) {
411                    if (sourcetokenlist.get(li).issolidtoken()) { lastSolid = sourcetokenlist.get(li); break; }
412                }
413                if ((lastSolid != null)
414                        && ((lastSolid.tokentype == ETokenType.ttidentifier)
415                            || (lastSolid.tokencode == ')') || (lastSolid.tokencode == ']'))) {
416                    int line = (int) asourcetoken.lineNo;
417                    int col = (int) asourcetoken.columnNo;
418                    sourcetokenlist.add(createSyntheticToken(".", TBaseType.rrw_clickhouse_tuple_dot, ETokenType.ttperiod, line, col));
419                    TSourceToken num = createSyntheticToken(asourcetoken.toString().substring(1), TBaseType.iconst, ETokenType.ttnumber, line, col + 1);
420                    sourcetokenlist.add(num);
421                    asourcetoken = getanewsourcetoken();
422                    if (asourcetoken == null) break;
423                    yychar = asourcetoken.tokencode;
424                    continue;
425                }
426            }
427
428            sourcetokenlist.add(asourcetoken);
429
430            asourcetoken = getanewsourcetoken();
431            if (asourcetoken == null) break;
432
433            yychar = asourcetoken.tokencode;
434        }
435
436        clickhousePostTokenAdjust();
437    }
438
439    /**
440     * Post-tokenize adjustments that need lookahead/lookbehind over the whole
441     * list. Three rewrites, all ClickHouse-specific:
442     *
443     * 1. IN whose next solid token is PARTITION becomes RW_IN_BEFORE_PARTITION.
444     *    ALTER TABLE t UPDATE a = 1 IN PARTITION p WHERE ... is LALR(1)-blocked
445     *    with plain IN: "a = 1 IN (...)" is a legal value expression, so the
446     *    parser must shift IN into the expression and then dies at PARTITION.
447     *    "x IN PARTITION" is never a legal expression, so the rewrite is safe
448     *    globally; the grammar accepts both spellings via ch_in_before_partition.
449     *
450     * 2. FROM/TO demoted to identifiers where the keyword reading is impossible:
451     *    after '.' (g.from), or when the next solid token is ',', ';' or ')'
452     *    (SELECT from, to, label ... ORDER BY from). ClickHouse itself accepts
453     *    these as column names (WITH RECURSIVE documentation examples).
454     *
455     * 3. INSERT INTO t FORMAT <name> followed by inline data rows: everything
456     *    after the format name up to the statement's ';' is raw data in the
457     *    named format, not SQL. It becomes one sqlpluscmd token holding the
458     *    text as written, which TCustomParser already skips (same mechanism as
459     *    handleCreateTableErrorRecovery above). Skipped when a SELECT comes
460     *    before FORMAT: that FORMAT belongs to the query.
461     */
462    private void clickhousePostTokenAdjust() {
463        parallelWithSeparators.clear();
464        foldedRowTokens.clear();
465        // Token-list surgery above (the :: CAST rewrite and the bind/float
466        // splits) inserts or removes tokens, but the lexer stamps posinlist
467        // from its own running count, so every token after the first edit
468        // carries a stale position. The statement splitter walks tokens BY
469        // posinlist (nextsolidtoken), so a stale value makes it read the
470        // wrong token and mistype the NEXT statement as sstinvalid — this
471        // predates the splits: SELECT a::UInt32; CREATE TABLE ... already
472        // mistyped the CREATE. Renumber once so positions match reality.
473        for (int ri = 0; ri < sourcetokenlist.size(); ri++) {
474            sourcetokenlist.get(ri).posinlist = ri;
475        }
476
477        TSourceToken prevSolid = null;
478        for (int i = 0; i < sourcetokenlist.size(); i++) {
479            TSourceToken st = sourcetokenlist.get(i);
480            if (!st.issolidtoken()) continue;
481
482            if (st.tokencode == TBaseType.rrw_clickhouse_in) {
483                TSourceToken nx = nextSolidToken(i);
484                if ((nx != null) && (nx.tokencode == TBaseType.rrw_clickhouse_partition)) {
485                    st.tokencode = TBaseType.rrw_clickhouse_in_before_partition;
486                }
487            } else if ((st.tokencode == TBaseType.rrw_from) || (st.tokencode == TBaseType.rrw_to)) {
488                TSourceToken nx = nextSolidToken(i);
489                boolean afterDot = (prevSolid != null) && (prevSolid.tokencode == '.');
490                boolean beforeCloser = (nx != null)
491                        && ((nx.tokencode == ',') || (nx.tokencode == ';') || (nx.tokencode == ')'));
492                if (afterDot || beforeCloser) {
493                    st.tokencode = TBaseType.ident;
494                    st.tokentype = ETokenType.ttidentifier;
495                }
496            } else if (st.tokencode == TBaseType.rrw_clickhouse_at) {
497                // AT is only a keyword in "AT TIME ZONE" / "AT LOCAL"; anywhere
498                // else it stays a plain identifier so aliases and columns named
499                // "at" keep parsing exactly as before AT was tokenized.
500                TSourceToken nx = nextSolidToken(i);
501                if ((nx == null) || ((nx.tokencode != TBaseType.rrw_clickhouse_time)
502                        && (nx.tokencode != TBaseType.rrw_clickhouse_local))) {
503                    st.tokencode = TBaseType.ident;
504                    st.tokentype = ETokenType.ttidentifier;
505                }
506            } else if (st.tokencode == TBaseType.rrw_clickhouse_zone) {
507                // ZONE is only a keyword directly after TIME.
508                if ((prevSolid == null) || (prevSolid.tokencode != TBaseType.rrw_clickhouse_time)) {
509                    st.tokencode = TBaseType.ident;
510                    st.tokentype = ETokenType.ttidentifier;
511                }
512            } else if (st.tokencode == '.') {
513                TSourceToken nx = nextSolidToken(i);
514                boolean prevOk = (prevSolid != null)
515                        && ((prevSolid.tokentype == ETokenType.ttidentifier)
516                            || (prevSolid.tokencode == ')') || (prevSolid.tokencode == ']'));
517                if (prevOk && (nx != null) && (nx.tokencode == TBaseType.iconst)) {
518                    st.tokencode = TBaseType.rrw_clickhouse_tuple_dot;
519                }
520            } else if (st.tokencode == TBaseType.rrw_insert) {
521                rewriteInsertFormatRawData(i);
522            } else if ((st.tokencode == TBaseType.ident) && st.toString().equalsIgnoreCase("PARALLEL")) {
523                // stmt1 PARALLEL WITH stmt2: an execution combinator GSP does not
524                // model. PARALLEL and WITH become a separator between the two
525                // statements, which both parse with full ASTs; the concurrency
526                // annotation itself is not modelled. Guarded on the token after
527                // WITH being a statement-starting keyword, so ORDER BY parallel
528                // WITH FILL, LIMIT parallel WITH TIES and GROUP BY parallel WITH
529                // TOTALS stay untouched.
530                TSourceToken w = nextSolidToken(i);
531                if ((w != null) && (w.tokencode == TBaseType.rrw_with)) {
532                    TSourceToken head = nextSolidToken(w.posinlist);
533                    if ((head != null) && isClickhouseStatementStart(head.tokencode)) {
534                        // The parser skips sqlpluscmd, and the splitter ends stmt1
535                        // at the pair and gives it to neither statement, so each
536                        // statement's text ("DROP TABLE t1", "DROP TABLE t2;")
537                        // re-parses on its own. The words stay keywords with their
538                        // text, so a formatter prints and cases them and a content
539                        // check compares them; written as ';' they came out of
540                        // every formatter as "; ;".
541                        st.tokencode = TBaseType.sqlpluscmd;
542                        st.tokentype = ETokenType.ttkeyword;
543                        w.tokencode = TBaseType.sqlpluscmd;
544                        w.tokentype = ETokenType.ttkeyword;
545                        parallelWithSeparators.add(st);
546                        parallelWithSeparators.add(w);
547                    }
548                }
549            }
550
551            prevSolid = st;
552        }
553        dropFoldedRowTokens();
554    }
555
556    private boolean isClickhouseStatementStart(int tokencode) {
557        switch (tokencode) {
558            case TBaseType.rrw_select:
559            case TBaseType.rrw_insert:
560            case TBaseType.rrw_delete:
561            case TBaseType.rrw_create:
562            case TBaseType.rrw_drop:
563            case TBaseType.rrw_truncate:
564            case 304:  // UPDATE
565            case 426:  // ALTER
566            case 476:  // RENAME
567            case 504:  // OPTIMIZE
568            case 1181: // DETACH
569            case 1182: // ATTACH
570                return true;
571            default:
572                return false;
573        }
574    }
575
576    private TSourceToken nextSolidToken(int from) {
577        for (int j = from + 1; j < sourcetokenlist.size(); j++) {
578            TSourceToken t = sourcetokenlist.get(j);
579            if (t.issolidtoken()) return t;
580        }
581        return null;
582    }
583
584    private void rewriteInsertFormatRawData(int insertPos) {
585        // The insert's FORMAT clause is "FORMAT <name>" after the target, outside parentheses, so a
586        // column, table, database or setting named format is not it: "(a, format)", "INTO db.format",
587        // "SETTINGS format = 1". VALUES, SELECT or WITH start the data or the query instead; a FORMAT
588        // after them is the query's.
589        int formatPos = -1;
590        int depth = 0;
591        TSourceToken prev = null;
592        for (int j = afterInsertTarget(insertPos); j < sourcetokenlist.size(); j++) {
593            TSourceToken t = sourcetokenlist.get(j);
594            if (!t.issolidtoken()) continue;
595            if (t.tokencode == ';') return;
596            if (t.tokencode == TBaseType.rrw_select) return;
597            if (t.tokencode == '(') {
598                depth++;
599            } else if (t.tokencode == ')') {
600                if (depth > 0) depth--;
601            } else if ((depth == 0) && !isName(prev, j)) {
602                if ((t.tokencode == TBaseType.rrw_values) || (t.tokencode == TBaseType.rrw_with)) return;
603                if (t.tokencode == TBaseType.rrw_clickhouse_format) {
604                    TSourceToken name = nextSolidToken(j);
605                    if ((name != null) && !isPunctuation(name)) {
606                        formatPos = j;
607                        break;
608                    }
609                }
610            }
611            prev = t;
612        }
613        if (formatPos < 0) return;
614        int namePos = nextSolidToken(formatPos).posinlist;
615        // Everything after "FORMAT <name>" is data to ClickHouse, a row reading "select" or only a
616        // comment or a blank line included, so the end is found before looking at what the rows hold
617        int end = sourcetokenlist.size();
618        for (int j = namePos + 1; j < sourcetokenlist.size(); j++) {
619            TSourceToken t = sourcetokenlist.get(j);
620            if (t.tokencode == ';') {
621                end = j;
622                break;
623            }
624            if (t.tokencode == TBaseType.cmtdoublehyphen) {
625                // "--" or "#" in the rows is data, not a comment: a ';' in it
626                // still ends the rows ("1,--none--;"), as it did when "--x"
627                // lexed as two minus signs
628                int semi = t.toString().indexOf(';');
629                if (semi > 0) {
630                    splitRawRowsAt(j, semi);
631                    end = j + 1;
632                    break;
633                }
634            }
635        }
636        // ClickHouse reads the data from after the format name, past spaces, tabs and form feeds and
637        // then one line break. Those blanks stay whitespace; the rest is one token holding the rows as
638        // written, line breaks, tabs and case included, so a formatter cannot join or re-case them
639        // ("1,a\n2,b" came out as the single row "1,a 2,b")
640        int start = namePos + 1;
641        while ((start < end) && isBlankBeforeRows(sourcetokenlist.get(start))) {
642            start++;
643        }
644        if (start >= end) return; // no rows: "FORMAT CSV;"
645        TSourceToken rows = sourcetokenlist.get(start);
646        StringBuilder text = new StringBuilder();
647        for (int k = start; k < end; k++) {
648            text.append(sourcetokenlist.get(k).toString());
649        }
650        rows.setString(text.toString());
651        rows.tokencode = TBaseType.sqlpluscmd;
652        rows.tokentype = ETokenType.ttsqlpluscmd;
653        for (int k = start + 1; k < end; k++) {
654            // dropped in one pass at the end of the pre-pass; its rules skip sqlpluscmd until then
655            TSourceToken t = sourcetokenlist.get(k);
656            t.tokencode = TBaseType.sqlpluscmd;
657            foldedRowTokens.add(t);
658        }
659    }
660
661    /** Drop the tokens folded into their rows' token, in one pass, and renumber the list. */
662    private void dropFoldedRowTokens() {
663        if (foldedRowTokens.isEmpty()) return;
664        int kept = 0;
665        for (int k = 0; k < sourcetokenlist.size(); k++) {
666            TSourceToken t = sourcetokenlist.get(k);
667            if (foldedRowTokens.contains(t)) continue;
668            t.posinlist = kept;
669            sourcetokenlist.set(kept++, t);
670        }
671        for (int k = sourcetokenlist.size() - 1; k >= kept; k--) {
672            sourcetokenlist.remove(k);
673        }
674        foldedRowTokens.clear();
675    }
676
677    /**
678     * The index after the target of the INSERT at {@code insertPos}: "INTO [TABLE | FUNCTION] name", the
679     * name's parts joined by '.', so a target named format, values, table or into ("INTO db.table FORMAT
680     * CSV") is never read as a keyword. The index after INSERT when no INTO follows.
681     */
682    private int afterInsertTarget(int insertPos) {
683        TSourceToken into = nextSolidToken(insertPos);
684        if ((into == null) || (into.tokencode != TBaseType.rrw_into)) return insertPos + 1;
685        TSourceToken part = nextSolidToken(into.posinlist);
686        if (part == null) return into.posinlist + 1;
687        if ((part.tokencode == TBaseType.rrw_table) || (part.tokencode == TBaseType.rrw_function)) {
688            TSourceToken after = nextSolidToken(part.posinlist);
689            // the keyword, unless it is itself a database name ("INTO table.t")
690            if ((after != null) && (after.tokencode != '.')) part = after;
691        }
692        if (isPunctuation(part)) return part.posinlist;
693        for (TSourceToken dot = nextSolidToken(part.posinlist); (dot != null) && (dot.tokencode == '.');
694                dot = nextSolidToken(part.posinlist)) {
695            TSourceToken next = nextSolidToken(dot.posinlist);
696            if (next == null) break;
697            part = next;
698        }
699        return part.posinlist + 1;
700    }
701
702    /**
703     * Whether the word at {@code pos}, after the solid token {@code prev}, is used as a name: a part of a
704     * dotted name, a setting ("format = 1") or a list item, not the keyword it is spelled like.
705     */
706    private boolean isName(TSourceToken prev, int pos) {
707        if ((prev != null) && (prev.tokencode == '.')) return true;
708        TSourceToken next = nextSolidToken(pos);
709        return (next != null) && ((next.tokencode == '.') || (next.tokencode == '=') || (next.tokencode == ',')
710                || (next.tokencode == ')'));
711    }
712
713    private static boolean isPunctuation(TSourceToken t) {
714        switch (t.tokentype) {
715            case ttsemicolon:
716            case ttcomma:
717            case ttperiod:
718            case ttequals:
719            case ttleftparenthesis:
720            case ttrightparenthesis:
721                return true;
722            default:
723                return false;
724        }
725    }
726
727    /** A whitespace token made only of the blanks ClickHouse skips before the first line break. */
728    private static boolean isBlankBeforeRows(TSourceToken t) {
729        if (t.tokentype != ETokenType.ttwhitespace) return false;
730        String s = t.toString();
731        for (int i = 0; i < s.length(); i++) {
732            char c = s.charAt(i);
733            if ((c != ' ') && (c != '\t') && (c != '\f')) return false;
734        }
735        return true;
736    }
737
738    /**
739     * Split the raw-row token at {@code j} at the ';' at index {@code semi} of its text: the text
740     * before stays in the rows, the ';' ends them, and the rest of the line is SQL again, lexed as
741     * such.
742     */
743    private void splitRawRowsAt(int j, int semi) {
744        TSourceToken t = sourcetokenlist.get(j);
745        String text = t.toString();
746        String rest = text.substring(semi + 1);
747        t.setString(text.substring(0, semi));
748        // no longer a comment that runs to the end of the line: a formatter
749        // prints it in place, before its ';', and does not re-case it
750        t.tokentype = ETokenType.ttsqlpluscmd;
751        TSourceToken semicolon = createSyntheticToken(";", ';', ETokenType.ttsemicolon, (int) t.lineNo,
752                (int) t.columnNo + semi);
753        semicolon.offset = t.offset + semi;
754        semicolon.setDbvendor(t.getDbvendor());
755        int at = j + 1;
756        sourcetokenlist.add(at++, semicolon);
757        if (!rest.isEmpty()) {
758            TGSqlParser sub = new TGSqlParser(EDbVendor.dbvclickhouse);
759            sub.sqltext = rest;
760            sub.tokenizeSqltext();
761            TSourceTokenList lexed = sub.getSourcetokenlist();
762            for (int k = 0; k < lexed.size(); k++) {
763                // the rest is one line: its offsets are columns after the ';'
764                TSourceToken c = lexed.get(k);
765                c.lineNo = t.lineNo;
766                c.columnNo = t.columnNo + semi + 1 + c.offset;
767                c.offset = t.offset + semi + 1 + c.offset;
768                c.container = sourcetokenlist;
769                c.setDbvendor(t.getDbvendor());
770                sourcetokenlist.add(at++, c);
771            }
772        }
773        for (int k = j; k < sourcetokenlist.size(); k++) {
774            sourcetokenlist.get(k).posinlist = k;
775        }
776    }
777
778    /**
779     * Check if next token is '(' for parameterized types and emit matching tokens,
780     * then close with ')' for CAST.
781     */
782    private void emitTypeParamsAndClose(int line, int col) {
783        TSourceToken peekToken = getanewsourcetoken();
784        if (peekToken != null && peekToken.tokencode == 40) { // '('
785            sourcetokenlist.add(peekToken);
786            int depth = 1;
787            while (depth > 0) {
788                TSourceToken innerToken = getanewsourcetoken();
789                if (innerToken == null) break;
790                sourcetokenlist.add(innerToken);
791                if (innerToken.tokencode == 40) depth++;
792                else if (innerToken.tokencode == 41) depth--;
793            }
794            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
795        } else {
796            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
797            // The peeked token must be put back into the stream;
798            // since we can't push back, check if it needs further processing
799            if (peekToken != null) {
800                // Check if the peeked token is also a typecast (chained casts like x::A::B)
801                if (peekToken.tokencode == TBaseType.typecast) {
802                    rewriteStandaloneTypecast(peekToken);
803                } else {
804                    sourcetokenlist.add(peekToken);
805                }
806            }
807        }
808    }
809
810    /**
811     * Create a synthetic token for CAST() rewriting.
812     */
813    private TSourceToken createSyntheticToken(String text, int tokencode, ETokenType tokentype, int lineNo, int columnNo) {
814        TSourceToken token = new TSourceToken(text);
815        token.tokencode = tokencode;
816        token.tokentype = tokentype;
817        token.tokenstatus = ETokenStatus.tsoriginal;
818        token.lineNo = lineNo;
819        token.columnNo = columnNo;
820        token.container = sourcetokenlist;
821        token.posinlist = sourcetokenlist.size();
822        return token;
823    }
824
825    /**
826     * Handle standalone :: typecast (lexer correctly tokenized :: as typecast,
827     * e.g., after string literals or closing parentheses).
828     * The expression before :: is already in sourcetokenlist.
829     * Rewrites: ... expr :: TypeName ... → ... CAST( expr AS TypeName ) ...
830     */
831    private void rewriteStandaloneTypecast(TSourceToken typecastToken) {
832        int line = (int) typecastToken.lineNo;
833        int col = (int) typecastToken.columnNo;
834
835        // Find the preceding expression's last solid token
836        int exprEndIdx = sourcetokenlist.size() - 1;
837        while (exprEndIdx >= 0) {
838            TSourceToken t = sourcetokenlist.get(exprEndIdx);
839            if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn
840                && t.tokentype != ETokenType.ttsimplecomment && t.tokentype != ETokenType.ttbracketedcomment) {
841                break;
842            }
843            exprEndIdx--;
844        }
845        if (exprEndIdx < 0) return;
846
847        // Find expression start
848        int exprStartIdx = exprEndIdx;
849        TSourceToken exprEndToken = sourcetokenlist.get(exprEndIdx);
850
851        if (exprEndToken.tokentype == ETokenType.ttrightparenthesis) {
852            // Walk back to find matching '('
853            int depth = 1;
854            int idx = exprEndIdx - 1;
855            while (idx >= 0 && depth > 0) {
856                TSourceToken t = sourcetokenlist.get(idx);
857                if (t.tokentype == ETokenType.ttrightparenthesis) depth++;
858                else if (t.tokentype == ETokenType.ttleftparenthesis) depth--;
859                idx--;
860            }
861            exprStartIdx = idx + 1; // idx+1 because we decremented one extra
862            // Check for function name before '('
863            if (exprStartIdx > 0) {
864                int fnIdx = exprStartIdx - 1;
865                while (fnIdx >= 0) {
866                    TSourceToken t = sourcetokenlist.get(fnIdx);
867                    if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn) {
868                        break;
869                    }
870                    fnIdx--;
871                }
872                if (fnIdx >= 0) {
873                    TSourceToken fnToken = sourcetokenlist.get(fnIdx);
874                    if (fnToken.tokentype == ETokenType.ttidentifier || fnToken.tokentype == ETokenType.ttkeyword) {
875                        exprStartIdx = fnIdx;
876                    }
877                }
878            }
879        }
880
881        // Collect expression tokens and whitespace
882        List<TSourceToken> beforeExpr = new ArrayList<>();
883        for (int i = 0; i < exprStartIdx; i++) {
884            beforeExpr.add(sourcetokenlist.get(i));
885        }
886        // Collect whitespace just before expression that should go before CAST
887        List<TSourceToken> wsBeforeExpr = new ArrayList<>();
888        while (!beforeExpr.isEmpty()) {
889            TSourceToken last = beforeExpr.get(beforeExpr.size() - 1);
890            if (last.tokentype == ETokenType.ttwhitespace || last.tokentype == ETokenType.ttreturn) {
891                wsBeforeExpr.add(0, beforeExpr.remove(beforeExpr.size() - 1));
892            } else {
893                break;
894            }
895        }
896
897        List<TSourceToken> exprTokens = new ArrayList<>();
898        for (int i = exprStartIdx; i <= exprEndIdx; i++) {
899            exprTokens.add(sourcetokenlist.get(i));
900        }
901        List<TSourceToken> afterExpr = new ArrayList<>();
902        for (int i = exprEndIdx + 1; i < sourcetokenlist.size(); i++) {
903            afterExpr.add(sourcetokenlist.get(i));
904        }
905
906        // Clear and rebuild sourcetokenlist
907        sourcetokenlist.clear();
908        sourcetokenlist.curpos = -1;
909
910        // Re-add tokens before expression (without trailing whitespace)
911        for (TSourceToken t : beforeExpr) {
912            t.posinlist = sourcetokenlist.size();
913            sourcetokenlist.add(t);
914        }
915
916        // Add whitespace before CAST
917        for (TSourceToken t : wsBeforeExpr) {
918            t.posinlist = sourcetokenlist.size();
919            sourcetokenlist.add(t);
920        }
921
922        // Emit CAST(
923        sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col));
924        sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col));
925
926        // Re-add expression tokens
927        for (TSourceToken t : exprTokens) {
928            t.posinlist = sourcetokenlist.size();
929            sourcetokenlist.add(t);
930        }
931
932        // Emit AS
933        sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col));
934
935        // Read type name and emit closing paren
936        TSourceToken typeToken = getanewsourcetoken();
937        if (typeToken == null) {
938            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
939            return;
940        }
941
942        // Skip whitespace
943        while (typeToken != null && (typeToken.tokentype == ETokenType.ttwhitespace || typeToken.tokentype == ETokenType.ttreturn)) {
944            typeToken = getanewsourcetoken();
945        }
946        if (typeToken == null) {
947            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
948            return;
949        }
950
951        sourcetokenlist.add(typeToken);
952
953        // Check for parameterized types and close
954        emitTypeParamsAndClose(line, col);
955    }
956
957    /**
958     * Get previous non-whitespace token.
959     */
960    private TSourceToken getprevsolidtoken(TSourceToken ptoken) {
961        TSourceToken ret = null;
962        TSourceTokenList lctokenlist = ptoken.container;
963
964        if (lctokenlist != null) {
965            if ((ptoken.posinlist > 0) && (lctokenlist.size() > ptoken.posinlist - 1)) {
966                if (!(
967                        (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttwhitespace)
968                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttreturn)
969                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttsimplecomment)
970                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttbracketedcomment)
971                )) {
972                    ret = lctokenlist.get(ptoken.posinlist - 1);
973                } else {
974                    ret = lctokenlist.nextsolidtoken(ptoken.posinlist - 1, -1, false);
975                }
976            }
977        }
978        return ret;
979    }
980
981    // ========== ClickHouse-Specific Raw Statement Extraction ==========
982
983    /**
984     * Extract raw ClickHouse SQL statements from tokenized source.
985     * <p>
986     * Simplified from MySQL - no stored procedure states,
987     * no DELIMITER command, no custom delimiters.
988     * ClickHouse uses semicolon as the only delimiter.
989     */
990    private void doclickhousegetrawsqlstatements(SqlParseResult.Builder builder) {
991        if (TBaseType.assigned(sqlstatements)) sqlstatements.clear();
992        if (!TBaseType.assigned(sourcetokenlist)) {
993            builder.sqlStatements(this.sqlstatements);
994            builder.errorCode(1);
995            builder.errorMessage("No source token list available");
996            return;
997        }
998
999        TCustomSqlStatement gcurrentsqlstatement = null;
1000        EFindSqlStateType gst = EFindSqlStateType.stnormal;
1001
1002        for (int i = 0; i < sourcetokenlist.size(); i++) {
1003            TSourceToken ast = sourcetokenlist.get(i);
1004            sourcetokenlist.curpos = i;
1005
1006            // Token transformations during raw statement extraction
1007            performRawStatementTokenTransformations(ast);
1008
1009            switch (gst) {
1010                case sterror: {
1011                    if (parallelWithSeparators.contains(ast)) {
1012                        // PARALLEL WITH: ends the statement without joining it
1013                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
1014                        appendToken(gcurrentsqlstatement, ast);
1015                        gst = EFindSqlStateType.stnormal;
1016                    } else if (ast.tokentype == ETokenType.ttsemicolon) {
1017                        appendToken(gcurrentsqlstatement, ast);
1018                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
1019                        gst = EFindSqlStateType.stnormal;
1020                    } else {
1021                        appendToken(gcurrentsqlstatement, ast);
1022                    }
1023                    break;
1024                }
1025
1026                case stnormal: {
1027                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
1028                            || (ast.tokencode == TBaseType.cmtslashstar)
1029                            || (ast.tokencode == TBaseType.lexspace)
1030                            || (ast.tokencode == TBaseType.lexnewline)
1031                            || (ast.tokentype == ETokenType.ttsemicolon)
1032                            || parallelWithSeparators.contains(ast)) {
1033                        if (gcurrentsqlstatement != null) {
1034                            appendToken(gcurrentsqlstatement, ast);
1035                        }
1036                        continue;
1037                    }
1038
1039                    // Find a token to start sql mode
1040                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);
1041
1042                    if (gcurrentsqlstatement != null) {
1043                        gst = EFindSqlStateType.stsql;
1044                        appendToken(gcurrentsqlstatement, ast);
1045                    } else {
1046                        // Error token found
1047                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo,
1048                                (ast.columnNo < 0 ? 0 : ast.columnNo),
1049                                "Error when tokenize", EErrorType.spwarning,
1050                                TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));
1051
1052                        ast.tokentype = ETokenType.tttokenlizererrortoken;
1053                        gst = EFindSqlStateType.sterror;
1054
1055                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
1056                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
1057                        appendToken(gcurrentsqlstatement, ast);
1058                    }
1059
1060                    break;
1061                }
1062
1063                case stsql: {
1064                    if (parallelWithSeparators.contains(ast)) {
1065                        // PARALLEL WITH (clickhousePostTokenAdjust): ends the
1066                        // statement without joining it, like the text between
1067                        // two statements.
1068                        gst = EFindSqlStateType.stnormal;
1069                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
1070                        appendToken(gcurrentsqlstatement, ast);
1071                        continue;
1072                    }
1073                    if (ast.tokentype == ETokenType.ttsemicolon) {
1074                        gst = EFindSqlStateType.stnormal;
1075                        appendToken(gcurrentsqlstatement, ast);
1076                        gcurrentsqlstatement.semicolonended = ast;
1077                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
1078                        continue;
1079                    }
1080
1081                    if (ast.tokencode == TBaseType.cmtdoublehyphen) {
1082                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)) {
1083                            gst = EFindSqlStateType.stnormal;
1084                            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
1085                            continue;
1086                        }
1087                    }
1088
1089                    appendToken(gcurrentsqlstatement, ast);
1090                    break;
1091                }
1092
1093                default:
1094                    break;
1095            }
1096        }
1097
1098        // Last statement
1099        if ((gcurrentsqlstatement != null) &&
1100                ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.sterror))) {
1101            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, true, builder);
1102        }
1103
1104        // Populate builder with results
1105        builder.sqlStatements(this.sqlstatements);
1106        builder.syntaxErrors(syntaxErrors instanceof ArrayList ?
1107                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
1108        builder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size());
1109    }
1110
1111    /**
1112     * Handle token transformations during raw statement extraction.
1113     * <p>
1114     * Handles MySQL-inherited token transformations for DATE, TIME, TIMESTAMP keywords
1115     * and other context-dependent token adjustments.
1116     */
1117    private void performRawStatementTokenTransformations(TSourceToken ast) {
1118        if (ast.tokencode == TBaseType.rrw_date) {
1119            TSourceToken st1 = ast.nextSolidToken();
1120            if (st1 != null) {
1121                if (st1.tokencode == '(') {
1122                    ast.tokencode = TBaseType.rrw_mysql_date_function;
1123                } else if (st1.tokencode == TBaseType.sconst) {
1124                    ast.tokencode = TBaseType.rrw_mysql_date_const;
1125                }
1126            }
1127        } else if (ast.tokencode == TBaseType.rrw_time) {
1128            TSourceToken st1 = ast.nextSolidToken();
1129            if (st1 != null) {
1130                if (st1.tokencode == TBaseType.sconst) {
1131                    ast.tokencode = TBaseType.rrw_mysql_time_const;
1132                }
1133            }
1134        } else if (ast.tokencode == TBaseType.rrw_timestamp) {
1135            TSourceToken st1 = ast.nextSolidToken();
1136            if (st1 != null) {
1137                if (st1.tokencode == TBaseType.sconst) {
1138                    ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
1139                } else if (st1.tokencode == TBaseType.ident) {
1140                    if (st1.toString().startsWith("\"")) {
1141                        ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
1142                        st1.tokencode = TBaseType.sconst;
1143                    }
1144                }
1145            }
1146        } else if (ast.tokencode == TBaseType.rrw_mysql_position) {
1147            TSourceToken st1 = ast.nextSolidToken();
1148            if (st1 != null) {
1149                if (st1.tokencode != '(') {
1150                    ast.tokencode = TBaseType.ident;
1151                }
1152            }
1153        } else if (ast.tokencode == TBaseType.rrw_interval) {
1154            TSourceToken leftParen = ast.searchToken('(', 1);
1155            if (leftParen != null) {
1156                int k = leftParen.posinlist + 1;
1157                int nested = 1;
1158                boolean commaToken = false;
1159                while (k < ast.container.size()) {
1160                    if (ast.container.get(k).tokencode == '(') {
1161                        nested++;
1162                    }
1163                    if (ast.container.get(k).tokencode == ')') {
1164                        nested--;
1165                        if (nested == 0) break;
1166                    }
1167                    if ((ast.container.get(k).tokencode == ',') && (nested == 1)) {
1168                        commaToken = true;
1169                        break;
1170                    }
1171                    k++;
1172                }
1173                if (commaToken) {
1174                    ast.tokencode = TBaseType.rrw_mysql_interval_func;
1175                }
1176            }
1177        }
1178    }
1179
1180    private void appendToken(TCustomSqlStatement statement, TSourceToken token) {
1181        if (statement == null || token == null) {
1182            return;
1183        }
1184        token.stmt = statement;
1185        statement.sourcetokenlist.add(token);
1186    }
1187
1188    @Override
1189    public String toString() {
1190        return "ClickhouseSqlParser{vendor=" + vendor + "}";
1191    }
1192}