package gudusoft.gsqlparser.parser;

import gudusoft.gsqlparser.EDbVendor;
import gudusoft.gsqlparser.TBaseType;
import gudusoft.gsqlparser.TCustomLexer;
import gudusoft.gsqlparser.TCustomParser;
import gudusoft.gsqlparser.TCustomSqlStatement;
import gudusoft.gsqlparser.TLexerClickhouse;
import gudusoft.gsqlparser.TParserClickhouse;
import gudusoft.gsqlparser.TSourceToken;
import gudusoft.gsqlparser.TSourceTokenList;
import gudusoft.gsqlparser.TStatementList;
import gudusoft.gsqlparser.TSyntaxError;
import gudusoft.gsqlparser.EFindSqlStateType;
import gudusoft.gsqlparser.ETokenType;
import gudusoft.gsqlparser.ETokenStatus;
import gudusoft.gsqlparser.ESqlStatementType;
import gudusoft.gsqlparser.EErrorType;
import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
import gudusoft.gsqlparser.compiler.TContext;
import gudusoft.gsqlparser.sqlenv.TSQLEnv;
import gudusoft.gsqlparser.compiler.TGlobalScope;
import gudusoft.gsqlparser.compiler.TFrame;

import java.util.ArrayList;
import java.util.List;
import java.util.Stack;

/**
 * ClickHouse database SQL parser implementation.
 *
 * <p>This parser handles ClickHouse-specific SQL syntax including:
 * <ul>
 *   <li>Standard SQL DML/DDL (SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, ALTER)</li>
 *   <li>ClickHouse-specific ENGINE clause in CREATE TABLE</li>
 *   <li>PREWHERE, FINAL, SAMPLE, ARRAY JOIN clauses</li>
 *   <li>FORMAT and SETTINGS clauses</li>
 *   <li>ClickHouse data types (Array, Tuple, Map, Nullable, LowCardinality, etc.)</li>
 * </ul>
 *
 * <p><b>Design Notes:</b>
 * <ul>
 *   <li>Extends {@link AbstractSqlParser}</li>
 *   <li>Based on MySQL grammar (ClickHouse shares backtick quoting, # comments, ENGINE clause)</li>
 *   <li>No stored procedure support (ClickHouse has no stored procedures)</li>
 *   <li>Delimiter character: ';' for SQL statements</li>
 * </ul>
 *
 * @see AbstractSqlParser
 * @see TLexerClickhouse
 * @see TParserClickhouse
 * @since 3.2.0.0
 */
public class ClickhouseSqlParser extends AbstractSqlParser {

    // ========== Lexer and Parser Instances ==========

    /** The ClickHouse lexer used for tokenization (public for TGSqlParser.getFlexer()) */
    public TLexerClickhouse flexer;
    private TParserClickhouse fparser;

    // ========== Constructor ==========

    /**
     * Construct ClickHouse SQL parser.
     * <p>
     * Configures the parser for ClickHouse database with default delimiter: semicolon (;)
     */
    public ClickhouseSqlParser() {
        super(EDbVendor.dbvclickhouse);

        // Set delimiter character - ClickHouse uses semicolon
        this.delimiterChar = ';';
        this.defaultDelimiterStr = ";";

        // Create lexer once - will be reused for all parsing operations
        this.flexer = new TLexerClickhouse();
        this.flexer.delimiterchar = this.delimiterChar;
        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;

        // CRITICAL: Set lexer for inherited getanewsourcetoken() method
        this.lexer = this.flexer;

        // Create parser once - will be reused for all parsing operations
        this.fparser = new TParserClickhouse(null);
        this.fparser.lexer = this.flexer;
    }

    // ========== AbstractSqlParser Abstract Methods Implementation ==========

    @Override
    protected TCustomLexer getLexer(ParserContext context) {
        return this.flexer;
    }

    @Override
    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
        this.fparser.sourcetokenlist = tokens;
        return this.fparser;
    }

    @Override
    protected void tokenizeVendorSql() {
        doclickhousetexttotokenlist();
    }

    @Override
    protected void setupVendorParsersForExtraction() {
        this.fparser.sqlcmds = this.sqlcmds;
        this.fparser.sourcetokenlist = this.sourcetokenlist;
    }

    @Override
    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
        doclickhousegetrawsqlstatements(builder);
    }

    @Override
    protected TStatementList performParsing(ParserContext context,
                                           TCustomParser parser,
                                           TCustomParser secondaryParser,
                                           TSourceTokenList tokens,
                                           TStatementList rawStatements) {
        this.sourcetokenlist = tokens;
        this.parserContext = context;
        this.sqlstatements = rawStatements;

        // Initialize sqlcmds for the parser
        this.sqlcmds = SqlCmdsFactory.get(vendor);
        this.fparser.sqlcmds = this.sqlcmds;

        // Initialize global context for statement parsing
        initializeGlobalContext();

        // Parse each statement
        for (int i = 0; i < sqlstatements.size(); i++) {
            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);

            try {
                stmt.setFrameStack(frameStack);
                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());

                // Handle error recovery for CREATE TABLE statements if enabled
                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
                    handleCreateTableErrorRecovery(stmt);
                }

                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
                    copyErrorsFromStatement(stmt);
                }
            } catch (Exception ex) {
                handleStatementParsingException(stmt, i, ex);
                continue;
            }
        }

        // Clean up frame stack
        if (globalFrame != null) {
            globalFrame.popMeFromStack(frameStack);
        }

        return this.sqlstatements;
    }

    /**
     * Handle error recovery for CREATE TABLE statements.
     */
    private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
        if ((stmt.sqlstatementtype != ESqlStatementType.sstcreatetable) || TBaseType.c_createTableStrictParsing) {
            return;
        }

        int nested = 0;
        boolean isIgnore = false, isFoundIgnoreToken = false;
        TSourceToken firstIgnoreToken = null;

        for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
            TSourceToken st = stmt.sourcetokenlist.get(k);
            if (isIgnore) {
                if (st.issolidtoken() && (st.tokencode != ';')) {
                    isFoundIgnoreToken = true;
                    if (firstIgnoreToken == null) {
                        firstIgnoreToken = st;
                    }
                }
                if (st.tokencode != ';') {
                    st.tokencode = TBaseType.sqlpluscmd;
                }
                continue;
            }
            if (st.tokencode == (int) ')') {
                nested--;
                if (nested == 0) {
                    boolean isSelect = false;
                    TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
                    if (st1 != null) {
                        TSourceToken st2 = st.searchToken((int) '(', 2);
                        if (st2 != null) {
                            TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
                            isSelect = (st3 != null);
                        }
                    }
                    if (!isSelect) isIgnore = true;
                }
            } else if (st.tokencode == (int) '(') {
                nested++;
            }
        }

        if (isFoundIgnoreToken) {
            stmt.clearError();
            stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree());
        }
    }

    // ========== ClickHouse-Specific Tokenization ==========

    /**
     * Perform ClickHouse-specific tokenization.
     * <p>
     * Based on MySQL tokenization but simplified - no DELIMITER command,
     * no stored procedure handling. Handles ClickHouse-specific token
     * transformations including :: typecast operator splitting.
     */
    private void doclickhousetexttotokenlist() {
        TSourceToken asourcetoken;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if (asourcetoken == null) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0) {
            // Handle :: typecast operator: the lexer produces mysqllabel (identifier:)
            // followed by bind_v (:name) for "identifier::TypeName". We need to
            // rewrite as CAST(identifier AS TypeName).
            if (yychar == TBaseType.mysqllabel) {
                // Append BEFORE fetching the lookahead: getanewsourcetoken's
                // whitespace merge reads sourcetokenlist.get(curpos), which is
                // out of bounds while this token is held back — "SELECT {a: b}"
                // (label then whitespace) crashed the whole tokenize and the
                // parser silently returned zero tokens. The bind_v rewrite
                // removes the token again below.
                sourcetokenlist.add(asourcetoken);
                TSourceToken nextToken = getanewsourcetoken();
                if (nextToken != null && nextToken.tokencode == TBaseType.bind_v) {
                    sourcetokenlist.remove(sourcetokenlist.size() - 1);
                    // We have mysqllabel + bind_v pattern = "ident:" + ":TypeName"
                    // Rewrite as: CAST ( identifier AS TypeName )
                    String labelText = asourcetoken.toString();
                    String identText = labelText.substring(0, labelText.length() - 1);
                    String bindText = nextToken.toString();
                    String typeText = bindText.substring(1);
                    int line = (int) asourcetoken.lineNo;
                    int col = (int) asourcetoken.columnNo;

                    // Check if preceding token is '.' (qualified name like t1.col::Type)
                    // If so, pull qualified name prefix into the CAST expression
                    List<TSourceToken> qualifiedPrefix = new ArrayList<>();
                    int lastIdx = sourcetokenlist.size() - 1;
                    // Skip trailing whitespace
                    while (lastIdx >= 0 && (sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttwhitespace
                            || sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttreturn)) {
                        lastIdx--;
                    }
                    if (lastIdx >= 0 && sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttperiod) {
                        // Walk back collecting name.name.name. pattern
                        int prefixEnd = lastIdx;
                        int idx = lastIdx;
                        while (idx >= 0) {
                            TSourceToken t = sourcetokenlist.get(idx);
                            if (t.tokentype == ETokenType.ttperiod) {
                                idx--;
                            } else if (t.tokentype == ETokenType.ttidentifier || t.tokentype == ETokenType.ttkeyword) {
                                idx--;
                                // Skip whitespace between identifier and dot
                                while (idx >= 0 && (sourcetokenlist.get(idx).tokentype == ETokenType.ttwhitespace
                                        || sourcetokenlist.get(idx).tokentype == ETokenType.ttreturn)) {
                                    idx--;
                                }
                                // Check if next non-ws token is another dot (continue) or stop
                                if (idx >= 0 && sourcetokenlist.get(idx).tokentype == ETokenType.ttperiod) {
                                    continue;
                                } else {
                                    break;
                                }
                            } else {
                                break;
                            }
                        }
                        int prefixStart = idx + 1;
                        // Collect prefix tokens and remove from sourcetokenlist
                        for (int i = prefixStart; i <= prefixEnd; i++) {
                            qualifiedPrefix.add(sourcetokenlist.get(i));
                        }
                        // Remove prefix tokens from end of sourcetokenlist
                        while (sourcetokenlist.size() > prefixStart) {
                            sourcetokenlist.remove(sourcetokenlist.size() - 1);
                        }
                    }

                    // Emit: CAST (
                    sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col));
                    sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col));

                    // Re-emit qualified prefix if any (e.g., "t1.")
                    for (TSourceToken prefixToken : qualifiedPrefix) {
                        prefixToken.posinlist = sourcetokenlist.size();
                        sourcetokenlist.add(prefixToken);
                    }

                    // Emit: identifier or integer
                    asourcetoken.setAstext(identText);
                    if (identText.matches("\\d+")) {
                        asourcetoken.tokencode = TBaseType.iconst;
                        asourcetoken.tokentype = ETokenType.ttnumber;
                    } else {
                        asourcetoken.tokencode = TBaseType.ident;
                        asourcetoken.tokentype = ETokenType.ttidentifier;
                    }
                    sourcetokenlist.add(asourcetoken);

                    // Emit: AS
                    sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col));

                    // Emit: TypeName
                    nextToken.setAstext(typeText);
                    nextToken.tokencode = TBaseType.ident;
                    nextToken.tokentype = ETokenType.ttidentifier;
                    sourcetokenlist.add(nextToken);

                    // Check for parameterized types: TypeName(params)
                    emitTypeParamsAndClose(line, col);

                    asourcetoken = getanewsourcetoken();
                    if (asourcetoken == null) break;
                    yychar = asourcetoken.tokencode;
                    continue;
                } else {
                    // Not followed by bind_v; the mysqllabel is already in the
                    // list, continue with the lookahead token.
                    if (nextToken == null) break;
                    asourcetoken = nextToken;
                    yychar = asourcetoken.tokencode;
                    continue;
                }
            }

            // Handle standalone :: typecast token (e.g., after string literals: '2024-01-15'::Date)
            if (yychar == TBaseType.typecast) {
                rewriteStandaloneTypecast(asourcetoken);
                asourcetoken = getanewsourcetoken();
                if (asourcetoken == null) break;
                yychar = asourcetoken.tokencode;
                continue;
            }

            if (yychar == TBaseType.bind_v) {
                // ClickHouse has no :name bind variables; a bind_v that
                // survived the :: rewrite above is a map-literal colon glued
                // to its value ({'key1':1}). Split into ':' plus the value.
                String bindText = asourcetoken.toString();
                int bLine = (int) asourcetoken.lineNo;
                int bCol = (int) asourcetoken.columnNo;
                sourcetokenlist.add(createSyntheticToken(":", ':', ETokenType.ttcolon, bLine, bCol));
                String valText = bindText.substring(1);
                boolean allDigits = valText.length() > 0;
                for (int ci = 0; ci < valText.length(); ci++) {
                    if (!Character.isDigit(valText.charAt(ci))) { allDigits = false; break; }
                }
                if (allDigits) {
                    sourcetokenlist.add(createSyntheticToken(valText, TBaseType.iconst, ETokenType.ttnumber, bLine, bCol + 1));
                } else {
                    sourcetokenlist.add(createSyntheticToken(valText, TBaseType.ident, ETokenType.ttidentifier, bLine, bCol + 1));
                }
                asourcetoken = getanewsourcetoken();
                if (asourcetoken == null) break;
                yychar = asourcetoken.tokencode;
                continue;
            }

            // Tuple element access a.2: the lexer greedily reads ".2" as a
            // float constant. When a float starting with '.' directly follows
            // an identifier or a closing bracket, split it into a dedicated
            // tuple-access dot token plus an integer constant. The dot gets
            // its own token (CH_TUPLE_DOT) rather than '.' so the grammar
            // rule cannot collide with qualified names like left.col.
            if ((yychar == TBaseType.fconst) && asourcetoken.toString().startsWith(".")
                    && asourcetoken.toString().indexOf('.', 1) < 0) {
                TSourceToken lastSolid = null;
                for (int li = sourcetokenlist.size() - 1; li >= 0; li--) {
                    if (sourcetokenlist.get(li).issolidtoken()) { lastSolid = sourcetokenlist.get(li); break; }
                }
                if ((lastSolid != null)
                        && ((lastSolid.tokentype == ETokenType.ttidentifier)
                            || (lastSolid.tokencode == ')') || (lastSolid.tokencode == ']'))) {
                    int line = (int) asourcetoken.lineNo;
                    int col = (int) asourcetoken.columnNo;
                    sourcetokenlist.add(createSyntheticToken(".", TBaseType.rrw_clickhouse_tuple_dot, ETokenType.ttperiod, line, col));
                    TSourceToken num = createSyntheticToken(asourcetoken.toString().substring(1), TBaseType.iconst, ETokenType.ttnumber, line, col + 1);
                    sourcetokenlist.add(num);
                    asourcetoken = getanewsourcetoken();
                    if (asourcetoken == null) break;
                    yychar = asourcetoken.tokencode;
                    continue;
                }
            }

            sourcetokenlist.add(asourcetoken);

            asourcetoken = getanewsourcetoken();
            if (asourcetoken == null) break;

            yychar = asourcetoken.tokencode;
        }

        clickhousePostTokenAdjust();
    }

    /**
     * Post-tokenize adjustments that need lookahead/lookbehind over the whole
     * list. Three rewrites, all ClickHouse-specific:
     *
     * 1. IN whose next solid token is PARTITION becomes RW_IN_BEFORE_PARTITION.
     *    ALTER TABLE t UPDATE a = 1 IN PARTITION p WHERE ... is LALR(1)-blocked
     *    with plain IN: "a = 1 IN (...)" is a legal value expression, so the
     *    parser must shift IN into the expression and then dies at PARTITION.
     *    "x IN PARTITION" is never a legal expression, so the rewrite is safe
     *    globally; the grammar accepts both spellings via ch_in_before_partition.
     *
     * 2. FROM/TO demoted to identifiers where the keyword reading is impossible:
     *    after '.' (g.from), or when the next solid token is ',', ';' or ')'
     *    (SELECT from, to, label ... ORDER BY from). ClickHouse itself accepts
     *    these as column names (WITH RECURSIVE documentation examples).
     *
     * 3. INSERT INTO t FORMAT <name> followed by inline data rows: everything
     *    after the format name up to the statement's ';' is raw data in the
     *    named format, not SQL. Those tokens are turned into sqlpluscmd, which
     *    TCustomParser already skips (same mechanism as
     *    handleCreateTableErrorRecovery above). Skipped when the tail starts
     *    with SELECT/WITH so a query source is never eaten.
     */
    private void clickhousePostTokenAdjust() {
        // Token-list surgery above (the :: CAST rewrite and the bind/float
        // splits) inserts or removes tokens, but the lexer stamps posinlist
        // from its own running count, so every token after the first edit
        // carries a stale position. The statement splitter walks tokens BY
        // posinlist (nextsolidtoken), so a stale value makes it read the
        // wrong token and mistype the NEXT statement as sstinvalid — this
        // predates the splits: SELECT a::UInt32; CREATE TABLE ... already
        // mistyped the CREATE. Renumber once so positions match reality.
        for (int ri = 0; ri < sourcetokenlist.size(); ri++) {
            sourcetokenlist.get(ri).posinlist = ri;
        }

        TSourceToken prevSolid = null;
        for (int i = 0; i < sourcetokenlist.size(); i++) {
            TSourceToken st = sourcetokenlist.get(i);
            if (!st.issolidtoken()) continue;

            if (st.tokencode == TBaseType.rrw_clickhouse_in) {
                TSourceToken nx = nextSolidToken(i);
                if ((nx != null) && (nx.tokencode == TBaseType.rrw_clickhouse_partition)) {
                    st.tokencode = TBaseType.rrw_clickhouse_in_before_partition;
                }
            } else if ((st.tokencode == TBaseType.rrw_from) || (st.tokencode == TBaseType.rrw_to)) {
                TSourceToken nx = nextSolidToken(i);
                boolean afterDot = (prevSolid != null) && (prevSolid.tokencode == '.');
                boolean beforeCloser = (nx != null)
                        && ((nx.tokencode == ',') || (nx.tokencode == ';') || (nx.tokencode == ')'));
                if (afterDot || beforeCloser) {
                    st.tokencode = TBaseType.ident;
                    st.tokentype = ETokenType.ttidentifier;
                }
            } else if (st.tokencode == TBaseType.rrw_clickhouse_at) {
                // AT is only a keyword in "AT TIME ZONE" / "AT LOCAL"; anywhere
                // else it stays a plain identifier so aliases and columns named
                // "at" keep parsing exactly as before AT was tokenized.
                TSourceToken nx = nextSolidToken(i);
                if ((nx == null) || ((nx.tokencode != TBaseType.rrw_clickhouse_time)
                        && (nx.tokencode != TBaseType.rrw_clickhouse_local))) {
                    st.tokencode = TBaseType.ident;
                    st.tokentype = ETokenType.ttidentifier;
                }
            } else if (st.tokencode == TBaseType.rrw_clickhouse_zone) {
                // ZONE is only a keyword directly after TIME.
                if ((prevSolid == null) || (prevSolid.tokencode != TBaseType.rrw_clickhouse_time)) {
                    st.tokencode = TBaseType.ident;
                    st.tokentype = ETokenType.ttidentifier;
                }
            } else if (st.tokencode == '.') {
                TSourceToken nx = nextSolidToken(i);
                boolean prevOk = (prevSolid != null)
                        && ((prevSolid.tokentype == ETokenType.ttidentifier)
                            || (prevSolid.tokencode == ')') || (prevSolid.tokencode == ']'));
                if (prevOk && (nx != null) && (nx.tokencode == TBaseType.iconst)) {
                    st.tokencode = TBaseType.rrw_clickhouse_tuple_dot;
                }
            } else if (st.tokencode == TBaseType.rrw_insert) {
                rewriteInsertFormatRawData(i);
            } else if ((st.tokencode == TBaseType.ident) && st.toString().equalsIgnoreCase("PARALLEL")) {
                // stmt1 PARALLEL WITH stmt2: an execution combinator GSP does not
                // model. Rewriting PARALLEL into a statement separator (and WITH
                // into an ignored token) lets both component statements parse
                // with full ASTs; the concurrency annotation itself is dropped.
                // Guarded on the token after WITH being a statement-starting
                // keyword, so ORDER BY parallel WITH FILL, LIMIT parallel WITH
                // TIES and GROUP BY parallel WITH TOTALS stay untouched.
                TSourceToken w = nextSolidToken(i);
                if ((w != null) && (w.tokencode == TBaseType.rrw_with)) {
                    TSourceToken head = nextSolidToken(w.posinlist);
                    if ((head != null) && isClickhouseStatementStart(head.tokencode)) {
                        // Text is rewritten too: a statement's reconstructed text
                        // includes its trailing separator, and a re-parse of
                        // "DROP TABLE t1 PARALLEL" (trailing keyword, no WITH to
                        // guard on) would fail. The combinator is dropped from
                        // the token text exactly where it is dropped from the AST.
                        st.setString(";");
                        st.tokencode = ';';
                        st.tokentype = ETokenType.ttsemicolon;
                        w.setString(";");
                        w.tokencode = ';';
                        w.tokentype = ETokenType.ttsemicolon;
                    }
                }
            }

            prevSolid = st;
        }
    }

    private boolean isClickhouseStatementStart(int tokencode) {
        switch (tokencode) {
            case TBaseType.rrw_select:
            case TBaseType.rrw_insert:
            case TBaseType.rrw_delete:
            case TBaseType.rrw_create:
            case TBaseType.rrw_drop:
            case TBaseType.rrw_truncate:
            case 304:  // UPDATE
            case 426:  // ALTER
            case 476:  // RENAME
            case 504:  // OPTIMIZE
            case 1181: // DETACH
            case 1182: // ATTACH
                return true;
            default:
                return false;
        }
    }

    private TSourceToken nextSolidToken(int from) {
        for (int j = from + 1; j < sourcetokenlist.size(); j++) {
            TSourceToken t = sourcetokenlist.get(j);
            if (t.issolidtoken()) return t;
        }
        return null;
    }

    private void rewriteInsertFormatRawData(int insertPos) {
        int formatPos = -1;
        for (int j = insertPos + 1; j < sourcetokenlist.size(); j++) {
            TSourceToken t = sourcetokenlist.get(j);
            if (!t.issolidtoken()) continue;
            if (t.tokencode == ';') return;
            if (t.tokencode == TBaseType.rrw_select) return; // INSERT ... SELECT: its FORMAT belongs to the query
            if (t.tokencode == TBaseType.rrw_clickhouse_format) { formatPos = j; break; }
        }
        if (formatPos < 0) return;
        int namePos = -1;
        for (int j = formatPos + 1; j < sourcetokenlist.size(); j++) {
            TSourceToken t = sourcetokenlist.get(j);
            if (!t.issolidtoken()) continue;
            if (t.tokencode == ';') return;
            namePos = j;
            break;
        }
        if (namePos < 0) return;
        TSourceToken first = nextSolidToken(namePos);
        if (first == null) return;
        if ((first.tokencode == ';') || (first.tokencode == TBaseType.rrw_select)) return;
        for (int j = namePos + 1; j < sourcetokenlist.size(); j++) {
            TSourceToken t = sourcetokenlist.get(j);
            if (t.tokencode == ';') break;
            if (t.issolidtoken()) {
                t.tokencode = TBaseType.sqlpluscmd;
            }
        }
    }

    /**
     * Check if next token is '(' for parameterized types and emit matching tokens,
     * then close with ')' for CAST.
     */
    private void emitTypeParamsAndClose(int line, int col) {
        TSourceToken peekToken = getanewsourcetoken();
        if (peekToken != null && peekToken.tokencode == 40) { // '('
            sourcetokenlist.add(peekToken);
            int depth = 1;
            while (depth > 0) {
                TSourceToken innerToken = getanewsourcetoken();
                if (innerToken == null) break;
                sourcetokenlist.add(innerToken);
                if (innerToken.tokencode == 40) depth++;
                else if (innerToken.tokencode == 41) depth--;
            }
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
        } else {
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
            // The peeked token must be put back into the stream;
            // since we can't push back, check if it needs further processing
            if (peekToken != null) {
                // Check if the peeked token is also a typecast (chained casts like x::A::B)
                if (peekToken.tokencode == TBaseType.typecast) {
                    rewriteStandaloneTypecast(peekToken);
                } else {
                    sourcetokenlist.add(peekToken);
                }
            }
        }
    }

    /**
     * Create a synthetic token for CAST() rewriting.
     */
    private TSourceToken createSyntheticToken(String text, int tokencode, ETokenType tokentype, int lineNo, int columnNo) {
        TSourceToken token = new TSourceToken(text);
        token.tokencode = tokencode;
        token.tokentype = tokentype;
        token.tokenstatus = ETokenStatus.tsoriginal;
        token.lineNo = lineNo;
        token.columnNo = columnNo;
        token.container = sourcetokenlist;
        token.posinlist = sourcetokenlist.size();
        return token;
    }

    /**
     * Handle standalone :: typecast (lexer correctly tokenized :: as typecast,
     * e.g., after string literals or closing parentheses).
     * The expression before :: is already in sourcetokenlist.
     * Rewrites: ... expr :: TypeName ... → ... CAST( expr AS TypeName ) ...
     */
    private void rewriteStandaloneTypecast(TSourceToken typecastToken) {
        int line = (int) typecastToken.lineNo;
        int col = (int) typecastToken.columnNo;

        // Find the preceding expression's last solid token
        int exprEndIdx = sourcetokenlist.size() - 1;
        while (exprEndIdx >= 0) {
            TSourceToken t = sourcetokenlist.get(exprEndIdx);
            if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn
                && t.tokentype != ETokenType.ttsimplecomment && t.tokentype != ETokenType.ttbracketedcomment) {
                break;
            }
            exprEndIdx--;
        }
        if (exprEndIdx < 0) return;

        // Find expression start
        int exprStartIdx = exprEndIdx;
        TSourceToken exprEndToken = sourcetokenlist.get(exprEndIdx);

        if (exprEndToken.tokentype == ETokenType.ttrightparenthesis) {
            // Walk back to find matching '('
            int depth = 1;
            int idx = exprEndIdx - 1;
            while (idx >= 0 && depth > 0) {
                TSourceToken t = sourcetokenlist.get(idx);
                if (t.tokentype == ETokenType.ttrightparenthesis) depth++;
                else if (t.tokentype == ETokenType.ttleftparenthesis) depth--;
                idx--;
            }
            exprStartIdx = idx + 1; // idx+1 because we decremented one extra
            // Check for function name before '('
            if (exprStartIdx > 0) {
                int fnIdx = exprStartIdx - 1;
                while (fnIdx >= 0) {
                    TSourceToken t = sourcetokenlist.get(fnIdx);
                    if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn) {
                        break;
                    }
                    fnIdx--;
                }
                if (fnIdx >= 0) {
                    TSourceToken fnToken = sourcetokenlist.get(fnIdx);
                    if (fnToken.tokentype == ETokenType.ttidentifier || fnToken.tokentype == ETokenType.ttkeyword) {
                        exprStartIdx = fnIdx;
                    }
                }
            }
        }

        // Collect expression tokens and whitespace
        List<TSourceToken> beforeExpr = new ArrayList<>();
        for (int i = 0; i < exprStartIdx; i++) {
            beforeExpr.add(sourcetokenlist.get(i));
        }
        // Collect whitespace just before expression that should go before CAST
        List<TSourceToken> wsBeforeExpr = new ArrayList<>();
        while (!beforeExpr.isEmpty()) {
            TSourceToken last = beforeExpr.get(beforeExpr.size() - 1);
            if (last.tokentype == ETokenType.ttwhitespace || last.tokentype == ETokenType.ttreturn) {
                wsBeforeExpr.add(0, beforeExpr.remove(beforeExpr.size() - 1));
            } else {
                break;
            }
        }

        List<TSourceToken> exprTokens = new ArrayList<>();
        for (int i = exprStartIdx; i <= exprEndIdx; i++) {
            exprTokens.add(sourcetokenlist.get(i));
        }
        List<TSourceToken> afterExpr = new ArrayList<>();
        for (int i = exprEndIdx + 1; i < sourcetokenlist.size(); i++) {
            afterExpr.add(sourcetokenlist.get(i));
        }

        // Clear and rebuild sourcetokenlist
        sourcetokenlist.clear();
        sourcetokenlist.curpos = -1;

        // Re-add tokens before expression (without trailing whitespace)
        for (TSourceToken t : beforeExpr) {
            t.posinlist = sourcetokenlist.size();
            sourcetokenlist.add(t);
        }

        // Add whitespace before CAST
        for (TSourceToken t : wsBeforeExpr) {
            t.posinlist = sourcetokenlist.size();
            sourcetokenlist.add(t);
        }

        // Emit CAST(
        sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col));
        sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col));

        // Re-add expression tokens
        for (TSourceToken t : exprTokens) {
            t.posinlist = sourcetokenlist.size();
            sourcetokenlist.add(t);
        }

        // Emit AS
        sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col));

        // Read type name and emit closing paren
        TSourceToken typeToken = getanewsourcetoken();
        if (typeToken == null) {
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
            return;
        }

        // Skip whitespace
        while (typeToken != null && (typeToken.tokentype == ETokenType.ttwhitespace || typeToken.tokentype == ETokenType.ttreturn)) {
            typeToken = getanewsourcetoken();
        }
        if (typeToken == null) {
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
            return;
        }

        sourcetokenlist.add(typeToken);

        // Check for parameterized types and close
        emitTypeParamsAndClose(line, col);
    }

    /**
     * Get previous non-whitespace token.
     */
    private TSourceToken getprevsolidtoken(TSourceToken ptoken) {
        TSourceToken ret = null;
        TSourceTokenList lctokenlist = ptoken.container;

        if (lctokenlist != null) {
            if ((ptoken.posinlist > 0) && (lctokenlist.size() > ptoken.posinlist - 1)) {
                if (!(
                        (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttwhitespace)
                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttreturn)
                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttsimplecomment)
                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttbracketedcomment)
                )) {
                    ret = lctokenlist.get(ptoken.posinlist - 1);
                } else {
                    ret = lctokenlist.nextsolidtoken(ptoken.posinlist - 1, -1, false);
                }
            }
        }
        return ret;
    }

    // ========== ClickHouse-Specific Raw Statement Extraction ==========

    /**
     * Extract raw ClickHouse SQL statements from tokenized source.
     * <p>
     * Simplified from MySQL - no stored procedure states,
     * no DELIMITER command, no custom delimiters.
     * ClickHouse uses semicolon as the only delimiter.
     */
    private void doclickhousegetrawsqlstatements(SqlParseResult.Builder builder) {
        if (TBaseType.assigned(sqlstatements)) sqlstatements.clear();
        if (!TBaseType.assigned(sourcetokenlist)) {
            builder.sqlStatements(this.sqlstatements);
            builder.errorCode(1);
            builder.errorMessage("No source token list available");
            return;
        }

        TCustomSqlStatement gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;

        for (int i = 0; i < sourcetokenlist.size(); i++) {
            TSourceToken ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            // Token transformations during raw statement extraction
            performRawStatementTokenTransformations(ast);

            switch (gst) {
                case sterror: {
                    if (ast.tokentype == ETokenType.ttsemicolon) {
                        appendToken(gcurrentsqlstatement, ast);
                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
                        gst = EFindSqlStateType.stnormal;
                    } else {
                        appendToken(gcurrentsqlstatement, ast);
                    }
                    break;
                }

                case stnormal: {
                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode == TBaseType.cmtslashstar)
                            || (ast.tokencode == TBaseType.lexspace)
                            || (ast.tokencode == TBaseType.lexnewline)
                            || (ast.tokentype == ETokenType.ttsemicolon)) {
                        if (gcurrentsqlstatement != null) {
                            appendToken(gcurrentsqlstatement, ast);
                        }
                        continue;
                    }

                    // Find a token to start sql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null) {
                        gst = EFindSqlStateType.stsql;
                        appendToken(gcurrentsqlstatement, ast);
                    } else {
                        // Error token found
                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo,
                                (ast.columnNo < 0 ? 0 : ast.columnNo),
                                "Error when tokenize", EErrorType.spwarning,
                                TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        appendToken(gcurrentsqlstatement, ast);
                    }

                    break;
                }

                case stsql: {
                    if (ast.tokentype == ETokenType.ttsemicolon) {
                        gst = EFindSqlStateType.stnormal;
                        appendToken(gcurrentsqlstatement, ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
                        continue;
                    }

                    if (ast.tokencode == TBaseType.cmtdoublehyphen) {
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)) {
                            gst = EFindSqlStateType.stnormal;
                            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
                            continue;
                        }
                    }

                    appendToken(gcurrentsqlstatement, ast);
                    break;
                }

                default:
                    break;
            }
        }

        // Last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.sterror))) {
            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, true, builder);
        }

        // Populate builder with results
        builder.sqlStatements(this.sqlstatements);
        builder.syntaxErrors(syntaxErrors instanceof ArrayList ?
                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
        builder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size());
    }

    /**
     * Handle token transformations during raw statement extraction.
     * <p>
     * Handles MySQL-inherited token transformations for DATE, TIME, TIMESTAMP keywords
     * and other context-dependent token adjustments.
     */
    private void performRawStatementTokenTransformations(TSourceToken ast) {
        if (ast.tokencode == TBaseType.rrw_date) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode == '(') {
                    ast.tokencode = TBaseType.rrw_mysql_date_function;
                } else if (st1.tokencode == TBaseType.sconst) {
                    ast.tokencode = TBaseType.rrw_mysql_date_const;
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_time) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode == TBaseType.sconst) {
                    ast.tokencode = TBaseType.rrw_mysql_time_const;
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_timestamp) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode == TBaseType.sconst) {
                    ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
                } else if (st1.tokencode == TBaseType.ident) {
                    if (st1.toString().startsWith("\"")) {
                        ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
                        st1.tokencode = TBaseType.sconst;
                    }
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_mysql_position) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode != '(') {
                    ast.tokencode = TBaseType.ident;
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_interval) {
            TSourceToken leftParen = ast.searchToken('(', 1);
            if (leftParen != null) {
                int k = leftParen.posinlist + 1;
                int nested = 1;
                boolean commaToken = false;
                while (k < ast.container.size()) {
                    if (ast.container.get(k).tokencode == '(') {
                        nested++;
                    }
                    if (ast.container.get(k).tokencode == ')') {
                        nested--;
                        if (nested == 0) break;
                    }
                    if ((ast.container.get(k).tokencode == ',') && (nested == 1)) {
                        commaToken = true;
                        break;
                    }
                    k++;
                }
                if (commaToken) {
                    ast.tokencode = TBaseType.rrw_mysql_interval_func;
                }
            }
        }
    }

    private void appendToken(TCustomSqlStatement statement, TSourceToken token) {
        if (statement == null || token == null) {
            return;
        }
        token.stmt = statement;
        statement.sourcetokenlist.add(token);
    }

    @Override
    public String toString() {
        return "ClickhouseSqlParser{vendor=" + vendor + "}";
    }
}