package gudusoft.gsqlparser.parser;

import gudusoft.gsqlparser.EDbVendor;
import gudusoft.gsqlparser.TBaseType;
import gudusoft.gsqlparser.TCustomLexer;
import gudusoft.gsqlparser.TCustomParser;
import gudusoft.gsqlparser.TCustomSqlStatement;
import gudusoft.gsqlparser.TLexerClickhouse;
import gudusoft.gsqlparser.TParserClickhouse;
import gudusoft.gsqlparser.TSourceToken;
import gudusoft.gsqlparser.TSourceTokenList;
import gudusoft.gsqlparser.TStatementList;
import gudusoft.gsqlparser.TSyntaxError;
import gudusoft.gsqlparser.EFindSqlStateType;
import gudusoft.gsqlparser.ETokenType;
import gudusoft.gsqlparser.ETokenStatus;
import gudusoft.gsqlparser.ESqlStatementType;
import gudusoft.gsqlparser.EErrorType;
import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
import gudusoft.gsqlparser.compiler.TContext;
import gudusoft.gsqlparser.sqlenv.TSQLEnv;
import gudusoft.gsqlparser.compiler.TGlobalScope;
import gudusoft.gsqlparser.compiler.TFrame;

import java.util.ArrayList;
import java.util.List;
import java.util.Stack;

/**
 * ClickHouse database SQL parser implementation.
 *
 * <p>This parser handles ClickHouse-specific SQL syntax including:
 * <ul>
 *   <li>Standard SQL DML/DDL (SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, ALTER)</li>
 *   <li>ClickHouse-specific ENGINE clause in CREATE TABLE</li>
 *   <li>PREWHERE, FINAL, SAMPLE, ARRAY JOIN clauses</li>
 *   <li>FORMAT and SETTINGS clauses</li>
 *   <li>ClickHouse data types (Array, Tuple, Map, Nullable, LowCardinality, etc.)</li>
 * </ul>
 *
 * <p><b>Design Notes:</b>
 * <ul>
 *   <li>Extends {@link AbstractSqlParser}</li>
 *   <li>Based on MySQL grammar (ClickHouse shares backtick quoting, # comments, ENGINE clause)</li>
 *   <li>No stored procedure support (ClickHouse has no stored procedures)</li>
 *   <li>Delimiter character: ';' for SQL statements</li>
 * </ul>
 *
 * @see AbstractSqlParser
 * @see TLexerClickhouse
 * @see TParserClickhouse
 * @since 3.2.0.0
 */
public class ClickhouseSqlParser extends AbstractSqlParser {

    // ========== Lexer and Parser Instances ==========

    /** The ClickHouse lexer used for tokenization (public for TGSqlParser.getFlexer()) */
    public TLexerClickhouse flexer;
    private TParserClickhouse fparser;

    // ========== Constructor ==========

    /**
     * Construct ClickHouse SQL parser.
     * <p>
     * Configures the parser for ClickHouse database with default delimiter: semicolon (;)
     */
    public ClickhouseSqlParser() {
        super(EDbVendor.dbvclickhouse);

        // Set delimiter character - ClickHouse uses semicolon
        this.delimiterChar = ';';
        this.defaultDelimiterStr = ";";

        // Create lexer once - will be reused for all parsing operations
        this.flexer = new TLexerClickhouse();
        this.flexer.delimiterchar = this.delimiterChar;
        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;

        // CRITICAL: Set lexer for inherited getanewsourcetoken() method
        this.lexer = this.flexer;

        // Create parser once - will be reused for all parsing operations
        this.fparser = new TParserClickhouse(null);
        this.fparser.lexer = this.flexer;
    }

    // ========== AbstractSqlParser Abstract Methods Implementation ==========

    @Override
    protected TCustomLexer getLexer(ParserContext context) {
        return this.flexer;
    }

    @Override
    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
        this.fparser.sourcetokenlist = tokens;
        return this.fparser;
    }

    @Override
    protected void tokenizeVendorSql() {
        doclickhousetexttotokenlist();
    }

    @Override
    protected void setupVendorParsersForExtraction() {
        this.fparser.sqlcmds = this.sqlcmds;
        this.fparser.sourcetokenlist = this.sourcetokenlist;
    }

    @Override
    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
        doclickhousegetrawsqlstatements(builder);
    }

    @Override
    protected TStatementList performParsing(ParserContext context,
                                           TCustomParser parser,
                                           TCustomParser secondaryParser,
                                           TSourceTokenList tokens,
                                           TStatementList rawStatements) {
        this.sourcetokenlist = tokens;
        this.parserContext = context;
        this.sqlstatements = rawStatements;

        // Initialize sqlcmds for the parser
        this.sqlcmds = SqlCmdsFactory.get(vendor);
        this.fparser.sqlcmds = this.sqlcmds;

        // Initialize global context for statement parsing
        initializeGlobalContext();

        // Parse each statement
        for (int i = 0; i < sqlstatements.size(); i++) {
            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);

            try {
                stmt.setFrameStack(frameStack);
                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());

                // Handle error recovery for CREATE TABLE statements if enabled
                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
                    handleCreateTableErrorRecovery(stmt);
                }

                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
                    copyErrorsFromStatement(stmt);
                }
            } catch (Exception ex) {
                handleStatementParsingException(stmt, i, ex);
                continue;
            }
        }

        // Clean up frame stack
        if (globalFrame != null) {
            globalFrame.popMeFromStack(frameStack);
        }

        return this.sqlstatements;
    }

    /**
     * Handle error recovery for CREATE TABLE statements.
     */
    private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
        if ((stmt.sqlstatementtype != ESqlStatementType.sstcreatetable) || TBaseType.c_createTableStrictParsing) {
            return;
        }

        int nested = 0;
        boolean isIgnore = false, isFoundIgnoreToken = false;
        TSourceToken firstIgnoreToken = null;

        for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
            TSourceToken st = stmt.sourcetokenlist.get(k);
            if (isIgnore) {
                if (st.issolidtoken() && (st.tokencode != ';')) {
                    isFoundIgnoreToken = true;
                    if (firstIgnoreToken == null) {
                        firstIgnoreToken = st;
                    }
                }
                if (st.tokencode != ';') {
                    st.tokencode = TBaseType.sqlpluscmd;
                }
                continue;
            }
            if (st.tokencode == (int) ')') {
                nested--;
                if (nested == 0) {
                    boolean isSelect = false;
                    TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
                    if (st1 != null) {
                        TSourceToken st2 = st.searchToken((int) '(', 2);
                        if (st2 != null) {
                            TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
                            isSelect = (st3 != null);
                        }
                    }
                    if (!isSelect) isIgnore = true;
                }
            } else if (st.tokencode == (int) '(') {
                nested++;
            }
        }

        if (isFoundIgnoreToken) {
            stmt.clearError();
            stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree());
        }
    }

    // ========== ClickHouse-Specific Tokenization ==========

    /**
     * Perform ClickHouse-specific tokenization.
     * <p>
     * Based on MySQL tokenization but simplified - no DELIMITER command,
     * no stored procedure handling. Handles ClickHouse-specific token
     * transformations including :: typecast operator splitting.
     */
    private void doclickhousetexttotokenlist() {
        TSourceToken asourcetoken;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if (asourcetoken == null) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0) {
            // Handle :: typecast operator: the lexer produces mysqllabel (identifier:)
            // followed by bind_v (:name) for "identifier::TypeName". We need to
            // rewrite as CAST(identifier AS TypeName).
            if (yychar == TBaseType.mysqllabel) {
                TSourceToken nextToken = getanewsourcetoken();
                if (nextToken != null && nextToken.tokencode == TBaseType.bind_v) {
                    // We have mysqllabel + bind_v pattern = "ident:" + ":TypeName"
                    // Rewrite as: CAST ( identifier AS TypeName )
                    String labelText = asourcetoken.toString();
                    String identText = labelText.substring(0, labelText.length() - 1);
                    String bindText = nextToken.toString();
                    String typeText = bindText.substring(1);
                    int line = (int) asourcetoken.lineNo;
                    int col = (int) asourcetoken.columnNo;

                    // Check if preceding token is '.' (qualified name like t1.col::Type)
                    // If so, pull qualified name prefix into the CAST expression
                    List<TSourceToken> qualifiedPrefix = new ArrayList<>();
                    int lastIdx = sourcetokenlist.size() - 1;
                    // Skip trailing whitespace
                    while (lastIdx >= 0 && (sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttwhitespace
                            || sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttreturn)) {
                        lastIdx--;
                    }
                    if (lastIdx >= 0 && sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttperiod) {
                        // Walk back collecting name.name.name. pattern
                        int prefixEnd = lastIdx;
                        int idx = lastIdx;
                        while (idx >= 0) {
                            TSourceToken t = sourcetokenlist.get(idx);
                            if (t.tokentype == ETokenType.ttperiod) {
                                idx--;
                            } else if (t.tokentype == ETokenType.ttidentifier || t.tokentype == ETokenType.ttkeyword) {
                                idx--;
                                // Skip whitespace between identifier and dot
                                while (idx >= 0 && (sourcetokenlist.get(idx).tokentype == ETokenType.ttwhitespace
                                        || sourcetokenlist.get(idx).tokentype == ETokenType.ttreturn)) {
                                    idx--;
                                }
                                // Check if next non-ws token is another dot (continue) or stop
                                if (idx >= 0 && sourcetokenlist.get(idx).tokentype == ETokenType.ttperiod) {
                                    continue;
                                } else {
                                    break;
                                }
                            } else {
                                break;
                            }
                        }
                        int prefixStart = idx + 1;
                        // Collect prefix tokens and remove from sourcetokenlist
                        for (int i = prefixStart; i <= prefixEnd; i++) {
                            qualifiedPrefix.add(sourcetokenlist.get(i));
                        }
                        // Remove prefix tokens from end of sourcetokenlist
                        while (sourcetokenlist.size() > prefixStart) {
                            sourcetokenlist.remove(sourcetokenlist.size() - 1);
                        }
                    }

                    // Emit: CAST (
                    sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col));
                    sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col));

                    // Re-emit qualified prefix if any (e.g., "t1.")
                    for (TSourceToken prefixToken : qualifiedPrefix) {
                        prefixToken.posinlist = sourcetokenlist.size();
                        sourcetokenlist.add(prefixToken);
                    }

                    // Emit: identifier or integer
                    asourcetoken.setAstext(identText);
                    if (identText.matches("\\d+")) {
                        asourcetoken.tokencode = TBaseType.iconst;
                        asourcetoken.tokentype = ETokenType.ttnumber;
                    } else {
                        asourcetoken.tokencode = TBaseType.ident;
                        asourcetoken.tokentype = ETokenType.ttidentifier;
                    }
                    sourcetokenlist.add(asourcetoken);

                    // Emit: AS
                    sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col));

                    // Emit: TypeName
                    nextToken.setAstext(typeText);
                    nextToken.tokencode = TBaseType.ident;
                    nextToken.tokentype = ETokenType.ttidentifier;
                    sourcetokenlist.add(nextToken);

                    // Check for parameterized types: TypeName(params)
                    emitTypeParamsAndClose(line, col);

                    asourcetoken = getanewsourcetoken();
                    if (asourcetoken == null) break;
                    yychar = asourcetoken.tokencode;
                    continue;
                } else {
                    // Not followed by bind_v, add mysqllabel normally
                    sourcetokenlist.add(asourcetoken);
                    if (nextToken == null) break;
                    asourcetoken = nextToken;
                    yychar = asourcetoken.tokencode;
                    continue;
                }
            }

            // Handle standalone :: typecast token (e.g., after string literals: '2024-01-15'::Date)
            if (yychar == TBaseType.typecast) {
                rewriteStandaloneTypecast(asourcetoken);
                asourcetoken = getanewsourcetoken();
                if (asourcetoken == null) break;
                yychar = asourcetoken.tokencode;
                continue;
            }

            sourcetokenlist.add(asourcetoken);

            asourcetoken = getanewsourcetoken();
            if (asourcetoken == null) break;

            yychar = asourcetoken.tokencode;
        }
    }

    /**
     * Check if next token is '(' for parameterized types and emit matching tokens,
     * then close with ')' for CAST.
     */
    private void emitTypeParamsAndClose(int line, int col) {
        TSourceToken peekToken = getanewsourcetoken();
        if (peekToken != null && peekToken.tokencode == 40) { // '('
            sourcetokenlist.add(peekToken);
            int depth = 1;
            while (depth > 0) {
                TSourceToken innerToken = getanewsourcetoken();
                if (innerToken == null) break;
                sourcetokenlist.add(innerToken);
                if (innerToken.tokencode == 40) depth++;
                else if (innerToken.tokencode == 41) depth--;
            }
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
        } else {
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
            // The peeked token must be put back into the stream;
            // since we can't push back, check if it needs further processing
            if (peekToken != null) {
                // Check if the peeked token is also a typecast (chained casts like x::A::B)
                if (peekToken.tokencode == TBaseType.typecast) {
                    rewriteStandaloneTypecast(peekToken);
                } else {
                    sourcetokenlist.add(peekToken);
                }
            }
        }
    }

    /**
     * Create a synthetic token for CAST() rewriting.
     */
    private TSourceToken createSyntheticToken(String text, int tokencode, ETokenType tokentype, int lineNo, int columnNo) {
        TSourceToken token = new TSourceToken(text);
        token.tokencode = tokencode;
        token.tokentype = tokentype;
        token.tokenstatus = ETokenStatus.tsoriginal;
        token.lineNo = lineNo;
        token.columnNo = columnNo;
        token.container = sourcetokenlist;
        token.posinlist = sourcetokenlist.size();
        return token;
    }

    /**
     * Handle standalone :: typecast (lexer correctly tokenized :: as typecast,
     * e.g., after string literals or closing parentheses).
     * The expression before :: is already in sourcetokenlist.
     * Rewrites: ... expr :: TypeName ... → ... CAST( expr AS TypeName ) ...
     */
    private void rewriteStandaloneTypecast(TSourceToken typecastToken) {
        int line = (int) typecastToken.lineNo;
        int col = (int) typecastToken.columnNo;

        // Find the preceding expression's last solid token
        int exprEndIdx = sourcetokenlist.size() - 1;
        while (exprEndIdx >= 0) {
            TSourceToken t = sourcetokenlist.get(exprEndIdx);
            if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn
                && t.tokentype != ETokenType.ttsimplecomment && t.tokentype != ETokenType.ttbracketedcomment) {
                break;
            }
            exprEndIdx--;
        }
        if (exprEndIdx < 0) return;

        // Find expression start
        int exprStartIdx = exprEndIdx;
        TSourceToken exprEndToken = sourcetokenlist.get(exprEndIdx);

        if (exprEndToken.tokentype == ETokenType.ttrightparenthesis) {
            // Walk back to find matching '('
            int depth = 1;
            int idx = exprEndIdx - 1;
            while (idx >= 0 && depth > 0) {
                TSourceToken t = sourcetokenlist.get(idx);
                if (t.tokentype == ETokenType.ttrightparenthesis) depth++;
                else if (t.tokentype == ETokenType.ttleftparenthesis) depth--;
                idx--;
            }
            exprStartIdx = idx + 1; // idx+1 because we decremented one extra
            // Check for function name before '('
            if (exprStartIdx > 0) {
                int fnIdx = exprStartIdx - 1;
                while (fnIdx >= 0) {
                    TSourceToken t = sourcetokenlist.get(fnIdx);
                    if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn) {
                        break;
                    }
                    fnIdx--;
                }
                if (fnIdx >= 0) {
                    TSourceToken fnToken = sourcetokenlist.get(fnIdx);
                    if (fnToken.tokentype == ETokenType.ttidentifier || fnToken.tokentype == ETokenType.ttkeyword) {
                        exprStartIdx = fnIdx;
                    }
                }
            }
        }

        // Collect expression tokens and whitespace
        List<TSourceToken> beforeExpr = new ArrayList<>();
        for (int i = 0; i < exprStartIdx; i++) {
            beforeExpr.add(sourcetokenlist.get(i));
        }
        // Collect whitespace just before expression that should go before CAST
        List<TSourceToken> wsBeforeExpr = new ArrayList<>();
        while (!beforeExpr.isEmpty()) {
            TSourceToken last = beforeExpr.get(beforeExpr.size() - 1);
            if (last.tokentype == ETokenType.ttwhitespace || last.tokentype == ETokenType.ttreturn) {
                wsBeforeExpr.add(0, beforeExpr.remove(beforeExpr.size() - 1));
            } else {
                break;
            }
        }

        List<TSourceToken> exprTokens = new ArrayList<>();
        for (int i = exprStartIdx; i <= exprEndIdx; i++) {
            exprTokens.add(sourcetokenlist.get(i));
        }
        List<TSourceToken> afterExpr = new ArrayList<>();
        for (int i = exprEndIdx + 1; i < sourcetokenlist.size(); i++) {
            afterExpr.add(sourcetokenlist.get(i));
        }

        // Clear and rebuild sourcetokenlist
        sourcetokenlist.clear();
        sourcetokenlist.curpos = -1;

        // Re-add tokens before expression (without trailing whitespace)
        for (TSourceToken t : beforeExpr) {
            t.posinlist = sourcetokenlist.size();
            sourcetokenlist.add(t);
        }

        // Add whitespace before CAST
        for (TSourceToken t : wsBeforeExpr) {
            t.posinlist = sourcetokenlist.size();
            sourcetokenlist.add(t);
        }

        // Emit CAST(
        sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col));
        sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col));

        // Re-add expression tokens
        for (TSourceToken t : exprTokens) {
            t.posinlist = sourcetokenlist.size();
            sourcetokenlist.add(t);
        }

        // Emit AS
        sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col));

        // Read type name and emit closing paren
        TSourceToken typeToken = getanewsourcetoken();
        if (typeToken == null) {
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
            return;
        }

        // Skip whitespace
        while (typeToken != null && (typeToken.tokentype == ETokenType.ttwhitespace || typeToken.tokentype == ETokenType.ttreturn)) {
            typeToken = getanewsourcetoken();
        }
        if (typeToken == null) {
            sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col));
            return;
        }

        sourcetokenlist.add(typeToken);

        // Check for parameterized types and close
        emitTypeParamsAndClose(line, col);
    }

    /**
     * Get previous non-whitespace token.
     */
    private TSourceToken getprevsolidtoken(TSourceToken ptoken) {
        TSourceToken ret = null;
        TSourceTokenList lctokenlist = ptoken.container;

        if (lctokenlist != null) {
            if ((ptoken.posinlist > 0) && (lctokenlist.size() > ptoken.posinlist - 1)) {
                if (!(
                        (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttwhitespace)
                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttreturn)
                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttsimplecomment)
                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttbracketedcomment)
                )) {
                    ret = lctokenlist.get(ptoken.posinlist - 1);
                } else {
                    ret = lctokenlist.nextsolidtoken(ptoken.posinlist - 1, -1, false);
                }
            }
        }
        return ret;
    }

    // ========== ClickHouse-Specific Raw Statement Extraction ==========

    /**
     * Extract raw ClickHouse SQL statements from tokenized source.
     * <p>
     * Simplified from MySQL - no stored procedure states,
     * no DELIMITER command, no custom delimiters.
     * ClickHouse uses semicolon as the only delimiter.
     */
    private void doclickhousegetrawsqlstatements(SqlParseResult.Builder builder) {
        if (TBaseType.assigned(sqlstatements)) sqlstatements.clear();
        if (!TBaseType.assigned(sourcetokenlist)) {
            builder.sqlStatements(this.sqlstatements);
            builder.errorCode(1);
            builder.errorMessage("No source token list available");
            return;
        }

        TCustomSqlStatement gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;

        for (int i = 0; i < sourcetokenlist.size(); i++) {
            TSourceToken ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            // Token transformations during raw statement extraction
            performRawStatementTokenTransformations(ast);

            switch (gst) {
                case sterror: {
                    if (ast.tokentype == ETokenType.ttsemicolon) {
                        appendToken(gcurrentsqlstatement, ast);
                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
                        gst = EFindSqlStateType.stnormal;
                    } else {
                        appendToken(gcurrentsqlstatement, ast);
                    }
                    break;
                }

                case stnormal: {
                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode == TBaseType.cmtslashstar)
                            || (ast.tokencode == TBaseType.lexspace)
                            || (ast.tokencode == TBaseType.lexnewline)
                            || (ast.tokentype == ETokenType.ttsemicolon)) {
                        if (gcurrentsqlstatement != null) {
                            appendToken(gcurrentsqlstatement, ast);
                        }
                        continue;
                    }

                    // Find a token to start sql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null) {
                        gst = EFindSqlStateType.stsql;
                        appendToken(gcurrentsqlstatement, ast);
                    } else {
                        // Error token found
                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo,
                                (ast.columnNo < 0 ? 0 : ast.columnNo),
                                "Error when tokenize", EErrorType.spwarning,
                                TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        appendToken(gcurrentsqlstatement, ast);
                    }

                    break;
                }

                case stsql: {
                    if (ast.tokentype == ETokenType.ttsemicolon) {
                        gst = EFindSqlStateType.stnormal;
                        appendToken(gcurrentsqlstatement, ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
                        continue;
                    }

                    if (ast.tokencode == TBaseType.cmtdoublehyphen) {
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)) {
                            gst = EFindSqlStateType.stnormal;
                            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
                            continue;
                        }
                    }

                    appendToken(gcurrentsqlstatement, ast);
                    break;
                }

                default:
                    break;
            }
        }

        // Last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.sterror))) {
            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, true, builder);
        }

        // Populate builder with results
        builder.sqlStatements(this.sqlstatements);
        builder.syntaxErrors(syntaxErrors instanceof ArrayList ?
                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
        builder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size());
    }

    /**
     * Handle token transformations during raw statement extraction.
     * <p>
     * Handles MySQL-inherited token transformations for DATE, TIME, TIMESTAMP keywords
     * and other context-dependent token adjustments.
     */
    private void performRawStatementTokenTransformations(TSourceToken ast) {
        if (ast.tokencode == TBaseType.rrw_date) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode == '(') {
                    ast.tokencode = TBaseType.rrw_mysql_date_function;
                } else if (st1.tokencode == TBaseType.sconst) {
                    ast.tokencode = TBaseType.rrw_mysql_date_const;
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_time) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode == TBaseType.sconst) {
                    ast.tokencode = TBaseType.rrw_mysql_time_const;
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_timestamp) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode == TBaseType.sconst) {
                    ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
                } else if (st1.tokencode == TBaseType.ident) {
                    if (st1.toString().startsWith("\"")) {
                        ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
                        st1.tokencode = TBaseType.sconst;
                    }
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_mysql_position) {
            TSourceToken st1 = ast.nextSolidToken();
            if (st1 != null) {
                if (st1.tokencode != '(') {
                    ast.tokencode = TBaseType.ident;
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_interval) {
            TSourceToken leftParen = ast.searchToken('(', 1);
            if (leftParen != null) {
                int k = leftParen.posinlist + 1;
                int nested = 1;
                boolean commaToken = false;
                while (k < ast.container.size()) {
                    if (ast.container.get(k).tokencode == '(') {
                        nested++;
                    }
                    if (ast.container.get(k).tokencode == ')') {
                        nested--;
                        if (nested == 0) break;
                    }
                    if ((ast.container.get(k).tokencode == ',') && (nested == 1)) {
                        commaToken = true;
                        break;
                    }
                    k++;
                }
                if (commaToken) {
                    ast.tokencode = TBaseType.rrw_mysql_interval_func;
                }
            }
        }
    }

    private void appendToken(TCustomSqlStatement statement, TSourceToken token) {
        if (statement == null || token == null) {
            return;
        }
        token.stmt = statement;
        statement.sourcetokenlist.add(token);
    }

    @Override
    public String toString() {
        return "ClickhouseSqlParser{vendor=" + vendor + "}";
    }
}