package gudusoft.gsqlparser.parser;

import gudusoft.gsqlparser.EDbVendor;
import gudusoft.gsqlparser.EFindSqlStateType;
import gudusoft.gsqlparser.EErrorType;
import gudusoft.gsqlparser.ESqlStatementType;
import gudusoft.gsqlparser.ETokenStatus;
import gudusoft.gsqlparser.ETokenType;
import gudusoft.gsqlparser.TBaseType;
import gudusoft.gsqlparser.TCustomLexer;
import gudusoft.gsqlparser.TCustomParser;
import gudusoft.gsqlparser.TCustomSqlStatement;
import gudusoft.gsqlparser.TLexerDatabricks;
import gudusoft.gsqlparser.TParserDatabricks;
import gudusoft.gsqlparser.TSourceToken;
import gudusoft.gsqlparser.TSourceTokenList;
import gudusoft.gsqlparser.TStatementList;
import gudusoft.gsqlparser.TSyntaxError;
import gudusoft.gsqlparser.TLog;
import gudusoft.gsqlparser.stmt.oracle.TSqlplusCmdStatement;
import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
import gudusoft.gsqlparser.compiler.TContext;
import gudusoft.gsqlparser.compiler.TGlobalScope;
import gudusoft.gsqlparser.sqlenv.TSQLEnv;
import gudusoft.gsqlparser.compiler.TFrame;
import gudusoft.gsqlparser.resolver.TSQLResolver;
import gudusoft.gsqlparser.compiler.TASTEvaluator;
import gudusoft.gsqlparser.nodes.TTypeName;
import gudusoft.gsqlparser.EDataType;

import java.util.Stack;
import java.util.ArrayList;

/**
 * Databricks SQL parser implementation.
 *
 * <p>This parser handles Databricks-specific SQL syntax including:
 * <ul>
 *   <li>Databricks SQL dialect and extensions</li>
 *   <li>Databricks PL/SQL blocks</li>
 *   <li>Special handling for VALUES keyword in INSERT statements</li>
 *   <li>Datatype casting with literals (e.g., DATE '2021-2-1')</li>
 * </ul>
 *
 * <p><b>Implementation Status:</b> MIGRATED
 * <ul>
 *   <li><b>Phase:</b> Complete migration from delegation to full AbstractSqlParser implementation</li>
 *   <li><b>Current:</b> Self-contained Databricks parser using AbstractSqlParser template</li>
 *   <li><b>Goal:</b> No delegation to legacy TGSqlParser</li>
 * </ul>
 *
 * @see SqlParser
 * @see AbstractSqlParser
 * @see TLexerDatabricks
 * @see TParserDatabricks
 * @since 3.2.0.0
 */
public class DatabricksSqlParser extends AbstractSqlParser {

    /**
     * Construct Databricks SQL parser.
     * <p>
     * Configures the parser for Databricks database with default delimiter: semicolon (;)
     * <p>
     * Following the original TGSqlParser pattern, the lexer and parser are
     * created once in the constructor and reused for all parsing operations.
     */
    public DatabricksSqlParser() {
        super(EDbVendor.dbvdatabricks);
        this.delimiterChar = ';';
        this.defaultDelimiterStr = ";";

        // Create lexer once - will be reused for all parsing operations
        this.flexer = new TLexerDatabricks();
        this.flexer.delimiterchar = this.delimiterChar;
        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;

        // Set parent's lexer reference for shared tokenization logic
        this.lexer = this.flexer;

        // Create parser once - will be reused for all parsing operations
        this.fparser = new TParserDatabricks(null);
        this.fparser.lexer = this.flexer;
    }

    // ========== Tokenization State (used during tokenization) ==========

    /** The Databricks lexer used for tokenization */
    public TLexerDatabricks flexer;

    // ========== Statement Parsing State (used during statement parsing) ==========

    /** Current statement being built */
    private TCustomSqlStatement gcurrentsqlstatement;

    /** SQL parser (for Databricks SQL statements) */
    private TParserDatabricks fparser;

    // ========== AbstractSqlParser Abstract Methods Implementation ==========

    /**
     * Return the Databricks lexer instance.
     * <p>
     * The lexer is created once in the constructor and reused for all
     * parsing operations.
     *
     * @param context parser context (not used, lexer already created)
     * @return the Databricks lexer instance created in constructor
     */
    @Override
    protected TCustomLexer getLexer(ParserContext context) {
        return this.flexer;
    }

    /**
     * Return the Databricks SQL parser instance with updated token list.
     * <p>
     * The parser is created once in the constructor and reused for all
     * parsing operations.
     *
     * @param context parser context (not used, parser already created)
     * @param tokens source token list to parse
     * @return the Databricks SQL parser instance created in constructor
     */
    @Override
    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
        this.fparser.sourcetokenlist = tokens;
        return this.fparser;
    }

    /**
     * Databricks uses a single parser, no secondary parser needed.
     *
     * @param context parser context
     * @param tokens source token list
     * @return null (no secondary parser)
     */
    @Override
    protected TCustomParser getSecondaryParser(ParserContext context, TSourceTokenList tokens) {
        return null;
    }

    /**
     * Hook method: Tokenize Databricks SQL by calling vendor-specific tokenization.
     */
    @Override
    protected void tokenizeVendorSql() {
        dodatabrickstexttotokenlist();
    }

    /**
     * Hook method: Setup parsers for raw statement extraction.
     * Inject sqlcmds and sourcetokenlist into parser.
     */
    @Override
    protected void setupVendorParsersForExtraction() {
        this.fparser.sqlcmds = this.sqlcmds;
        this.fparser.sourcetokenlist = this.sourcetokenlist;
    }

    /**
     * Hook method: Extract raw Databricks SQL statements.
     *
     * @param builder the result builder to populate
     */
    @Override
    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
        dodatabricksgetrawsqlstatements(builder);
    }

    // ========== Databricks-Specific Tokenization Logic ==========

    /**
     * Tokenize Databricks SQL text to token list.
     * <p>
     * This method processes the input SQL text and converts it into a sequence
     * of tokens. It handles Databricks-specific token processing including
     * MySQL-style comments and delimiter handling.
     * <p>
     * Migrated from TGSqlParser.dodatabrickstexttotokenlist() at line 4696.
     */
    private void dodatabrickstexttotokenlist() {
        TSourceToken asourcetoken, lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if (asourcetoken == null) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0) {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if (asourcetoken == null) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline) && (startDelimiter)) {
                startDelimiter = false;
                flexer.tmpDelimiter = sourcetokenlist.get(sourcetokenlist.size() - 1).getAstext();
            }

            yychar = asourcetoken.tokencode;
        }
    }

    /**
     * Check for MySQL-style comments in tokens.
     * <p>
     * This method is used to handle MySQL comment syntax which is also
     * supported by Databricks.
     *
     * @param asourcetoken the token to check
     */
    private void checkMySQLCommentToken(TSourceToken asourcetoken) {
        // MySQL comment handling - placeholder for now
        // The actual implementation would check for MySQL-style comments
        // This matches the pattern from TGSqlParser
    }

    // ========== Databricks-Specific Raw Statement Extraction Logic ==========

    /**
     * Extract raw SQL statements from token list for Databricks.
     * <p>
     * This method separates the token list into individual SQL statements
     * without performing full parsing. It handles Databricks-specific syntax:
     * <ul>
     *   <li>VALUES keyword disambiguation for INSERT statements</li>
     *   <li>Datatype casting with literals (DATE '2021-2-1')</li>
     *   <li>PL/SQL block detection with BEGIN/END</li>
     *   <li>Statement terminators (semicolons, slash, period)</li>
     * </ul>
     * <p>
     * Migrated from TGSqlParser.dodatabricksgetrawsqlstatements() at line 6944.
     *
     * @param builder the result builder to populate with raw statements
     */
    private void dodatabricksgetrawsqlstatements(SqlParseResult.Builder builder) {
        int waitingEnd = 0;
        boolean foundEnd = false;
        EDataType tmpDatatype = null;

        if (TBaseType.assigned(sqlstatements)) sqlstatements.clear();
        if (!TBaseType.assigned(sourcetokenlist)) {
            builder.errorCode(-1);
            builder.errorMessage("Source token list not assigned");
            return;
        }

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null, ast = null;

        for (int i = 0; i < sourcetokenlist.size(); i++) {

            if ((ast != null) && (ast.issolidtoken()))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            // Databricks-specific token adjustments
            if (ast.tokencode == TBaseType.rrw_values) {
                TSourceToken stParen = ast.searchToken('(', 1);
                if (stParen != null) {
                    TSourceToken stInsert = ast.searchToken(TBaseType.rrw_insert, -ast.posinlist, ';', true);
                    if (stInsert != null) {
                        TSourceToken stSemiColon = ast.searchToken(';', -ast.posinlist);
                        if ((stSemiColon != null) && (stSemiColon.posinlist > stInsert.posinlist)) {
                            // INSERT INTO test values (16,1), (8,2), (4,4), (2,0), (97, 16);
                            // VALUES (1);
                            // don't treat values(1) as insert values
                        } else {
                            TSourceToken stFrom = ast.searchToken(TBaseType.rrw_from, -ast.posinlist, ';', true);
                            if (stFrom != null) {
                                // don't treat values after from keyword as an insert values
                                // insert into inserttest values(10, 20, '40'), (-1, 2, DEFAULT),
                                // ((select 2), (select i from (values(3) ) as foo (i)), 'values are fun!');

                                // let check the INSERT keyword is close to VALUES than FROM keyword, if yes, treat it as insert values
                                if (stInsert.posinlist > stFrom.posinlist) {
                                    // https://www.sqlparser.com/bugs/mantisbt/view.php?id=3354
                                    ast.tokencode = TBaseType.rrw_databricks_values_insert;
                                }
                            } else {
                                ast.tokencode = TBaseType.rrw_databricks_values_insert;
                            }
                        }
                    }
                }
            } else if ((ast.tokencode == TBaseType.sconst) || (ast.tokencode == '+') || (ast.tokencode == '-')) {
                if ((lcprevsolidtoken != null) && (TTypeName.searchTypeByName(lcprevsolidtoken.toString()) != null)) {
                    // date '2021-2-1', turn date to TBaseType.rrw_databricks_datatype_used_to_cast
                    if (lcprevsolidtoken.tokencode != TBaseType.rrw_interval) {
                        lcprevsolidtoken.tokencode = TBaseType.rrw_databricks_datatype_used_to_cast;
                    }
                }
            } else if (ast.tokencode == TBaseType.rrw_databricks_POSITION) {
                // Databricks POSITION has two forms:
                //   1. POSITION(substr IN str)      -- SQL-standard form
                //   2. POSITION(substr, str[, pos]) -- comma form
                // The comma form parses fine as a generic function call, but the IN
                // form parses as func(<general IN predicate>), whose RHS (in_expr) is
                // restricted to scalars and cannot be a function/CASE/CAST. Route only
                // the IN form to the dedicated RW_POSITION_FUNCTION rule, which accepts
                // a full expression on both sides of IN. Leave the comma form untouched.
                if (isPositionInForm(ast)) {
                    ast.tokencode = TBaseType.rrw_databricks_POSITION_FUNCTION;
                }
            } else if ((ast.tokencode == TBaseType.ident) && ast.toString().equalsIgnoreCase("identifier")) {
                // Databricks IDENTIFIER(expr) as a DDL/DML write target:
                //   CREATE TABLE IDENTIFIER(:p || '_t') AS ...
                //   INSERT OVERWRITE IDENTIFIER(:cat || '.s.t') SELECT ...
                //   MERGE INTO IDENTIFIER(...) ...
                // In those positions the grammar expects a qualified_name, so the
                // generic function-call route used by FROM IDENTIFIER(...) is not
                // available. Route to the dedicated RW_IDENTIFIER_FUNC rule, but
                // only when (a) the previous solid token is TABLE/INTO/OVERWRITE/
                // EXISTS (write-target context; FROM and expression positions
                // already parse via the function-call path and stay untouched)
                // and (b) the argument starts with a parameter marker or string
                // literal - so a table literally named "identifier" followed by
                // a column-definition list is not captured. Mantis 4638.
                if (lcprevsolidtoken != null) {
                    String prevText = lcprevsolidtoken.toString();
                    // non-identifier-compare: matching SQL keywords (TABLE/INTO/OVERWRITE/EXISTS), not DB object names
                    if (prevText.equalsIgnoreCase("table") || prevText.equalsIgnoreCase("into")
                            || prevText.equalsIgnoreCase("overwrite") || prevText.equalsIgnoreCase("exists")) {
                        TSourceToken openParen = ast.nextSolidToken();
                        if ((openParen != null) && (openParen.tokencode == (int) '(')) {
                            TSourceToken firstArg = openParen.nextSolidToken();
                            if ((firstArg != null)
                                    && ((firstArg.tokencode == (int) ':') || (firstArg.tokencode == TBaseType.sconst))) {
                                ast.tokencode = TBaseType.rrw_databricks_identifier_func;
                            }
                        }
                    }
                }
            } else if (ast.tokencode == TBaseType.rrw_with) {
                // A plain RW_WITH introducing a schema/metrics clause poisons the
                // LALR tables, so route it to the synthetic RW_WITH_SCHEMA_OPTION
                // token when the following solid tokens spell one of:
                //   WITH SCHEMA {BINDING | COMPENSATION | EVOLUTION | TYPE EVOLUTION}
                //     (CREATE VIEW, Mantis 4644 / MERGE, Mantis 4645)
                //   WITH METRICS LANGUAGE ...   (metric views, Mantis 4643)
                // A CTE named "schema" or "metrics" is not captured: its name is
                // followed by AS or '(', never by BINDING/EVOLUTION/... or LANGUAGE.
                TSourceToken next1 = ast.nextSolidToken();
                TSourceToken next2 = (next1 == null) ? null : next1.nextSolidToken();
                if ((next1 != null) && (next2 != null)) {
                    String next1Text = next1.toString();
                    String next2Text = next2.toString();
                    // non-identifier-compare: matching SQL keywords of the schema/metrics clause, not DB object names
                    if (next1Text.equalsIgnoreCase("schema")
                            && (next2Text.equalsIgnoreCase("binding") || next2Text.equalsIgnoreCase("compensation")
                                || next2Text.equalsIgnoreCase("evolution"))) {
                        ast.tokencode = TBaseType.rrw_databricks_with_schema_option;
                    } else if (next1Text.equalsIgnoreCase("schema") && next2Text.equalsIgnoreCase("type")) {
                        // WITH SCHEMA TYPE is only valid as WITH SCHEMA TYPE EVOLUTION
                        TSourceToken next3 = next2.nextSolidToken();
                        // non-identifier-compare: matching SQL keywords of the schema clause, not DB object names
                        if ((next3 != null) && next3.toString().equalsIgnoreCase("evolution")) {
                            ast.tokencode = TBaseType.rrw_databricks_with_schema_option;
                        }
                    } else if (next1Text.equalsIgnoreCase("metrics") && next2Text.equalsIgnoreCase("language")) {
                        // the language name itself is left open (YAML today; new
                        // languages would otherwise need a parser change)
                        ast.tokencode = TBaseType.rrw_databricks_with_schema_option;
                    }
                }
            }

            switch (gst) {
                case sterror: {
                    if (ast.tokentype == ETokenType.ttsemicolon) {
                        appendToken(gcurrentsqlstatement, ast);
                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
                        gst = EFindSqlStateType.stnormal;
                    } else {
                        appendToken(gcurrentsqlstatement, ast);
                    }
                    break;
                } //sterror

                case stnormal: {
                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode == TBaseType.cmtslashstar)
                            || (ast.tokencode == TBaseType.lexspace)
                            || (ast.tokencode == TBaseType.lexnewline)
                            || (ast.tokentype == ETokenType.ttsemicolon)) {
                        if (gcurrentsqlstatement != null) {
                            appendToken(gcurrentsqlstatement, ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon)) {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon) {
                                // ;;;; continuous semicolon, treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode = TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }

                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null) {
                        if (gcurrentsqlstatement.isdatabricksplsql()) {
                            gst = EFindSqlStateType.ststoredprocedure;
                            appendToken(gcurrentsqlstatement, ast);
                            foundEnd = false;
                            if ((ast.tokencode == TBaseType.rrw_begin)
                                    || (ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package, 4) != null)) {
                                waitingEnd = 1;
                            }
                        } else {
                            gst = EFindSqlStateType.stsql;
                            appendToken(gcurrentsqlstatement, ast);
                        }
                    } else {
                        //error tokentext found
                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo, (ast.columnNo < 0 ? 0 : ast.columnNo)
                                , "Error when tokenlize", EErrorType.spwarning, TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        appendToken(gcurrentsqlstatement, ast);
                    }

                    break;
                } // stnormal

                case stsql: {
                    if (ast.tokentype == ETokenType.ttsemicolon) {
                        gst = EFindSqlStateType.stnormal;
                        appendToken(gcurrentsqlstatement, ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
                        continue;
                    }

                    if (sourcetokenlist.sqlplusaftercurtoken()) { //most probably is / cmd
                        gst = EFindSqlStateType.stnormal;
                        appendToken(gcurrentsqlstatement, ast);
                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
                        continue;
                    }
                    appendToken(gcurrentsqlstatement, ast);
                    break;
                }//case stsql

                case ststoredprocedure: {
                    if (ast.tokencode == TBaseType.rrw_begin) {
                        waitingEnd++;
                    } else if (ast.tokencode == TBaseType.rrw_if) {
                        if (ast.searchToken(TBaseType.rrw_end, -1) == null) {
                            //this is not if after END
                            waitingEnd++;
                        }
                    } else if (ast.tokencode == TBaseType.rrw_case) {
                        if (ast.searchToken(TBaseType.rrw_end, -1) == null) {
                            //this is not case after END
                            waitingEnd++;
                        }
                    } else if (ast.tokencode == TBaseType.rrw_loop) {
                        if (ast.searchToken(TBaseType.rrw_end, -1) == null) {
                            //this is not loop after END
                            waitingEnd++;
                        }
                    } else if (ast.tokencode == TBaseType.rrw_end) {
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) {
                            waitingEnd = 0;
                        }
                    }

                    if ((ast.tokentype == ETokenType.ttslash) && (ast.tokencode == TBaseType.sqlpluscmd)) {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(vendor);
                        appendToken(gcurrentsqlstatement, ast);
                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
                    } else if ((ast.tokentype == ETokenType.ttperiod) && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false))) {
                        // single dot at a separate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(vendor);
                        appendToken(gcurrentsqlstatement, ast);
                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
                    } else {
                        appendToken(gcurrentsqlstatement, ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0)
                                && (foundEnd)) {
                            gst = EFindSqlStateType.stnormal;
                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd) {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.getAstext());
                        if (m != 0) {
                            ast.tokencode = m;
                        } else {
                            ast.tokencode = TBaseType.ident;
                        }
                    }

                    break;
                } //ststoredprocedure
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) || (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                        (gst == EFindSqlStateType.sterror))) {
            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, true, builder);
        }

        // Populate builder with results
        builder.sqlStatements(this.sqlstatements);
        builder.syntaxErrors(syntaxErrors instanceof ArrayList ?
                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
        builder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size());
        if (!syntaxErrors.isEmpty()) {
            builder.errorMessage(String.format("Extraction completed with %d error(s)", syntaxErrors.size()));
        }
    }

    /**
     * Helper method to append token to statement.
     * Sets the token's stmt reference and adds it to the statement's token list.
     *
     * @param statement the statement to append to
     * @param token the token to append
     */
    private void appendToken(TCustomSqlStatement statement, TSourceToken token) {
        if (statement == null || token == null) {
            return;
        }
        token.stmt = statement;
        statement.sourcetokenlist.add(token);
    }

    // ========== Statement Parsing Logic ==========

    /**
     * Parse all raw statements to build AST.
     * <p>
     * This method iterates through all raw statements and calls parsestatement()
     * on each one to build the Abstract Syntax Tree. It handles error recovery
     * for CREATE TABLE statements and collects syntax errors.
     *
     * @param context parser context with configuration
     * @param parser primary parser instance
     * @param secondaryParser secondary parser (null for Databricks)
     * @param tokens source token list
     * @param rawStatements raw statements from extraction phase
     * @return statement list with parsed AST
     */
    @Override
    protected TStatementList performParsing(ParserContext context, TCustomParser parser,
                                           TCustomParser secondaryParser, TSourceTokenList tokens,
                                           TStatementList rawStatements) {
        // Store references
        this.fparser = (TParserDatabricks) parser;
        this.sourcetokenlist = tokens;
        this.parserContext = context;
        this.sqlstatements = rawStatements;

        // Initialize sqlcmds
        this.sqlcmds = SqlCmdsFactory.get(vendor);
        this.fparser.sqlcmds = this.sqlcmds;

        // Initialize global context (inherited from AbstractSqlParser)
        initializeGlobalContext();

        // Parse each statement
        for (int i = 0; i < sqlstatements.size(); i++) {
            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);
            try {
                stmt.setFrameStack(frameStack);
                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());

                // Vendor-specific post-processing (override hook if needed)
                afterStatementParsed(stmt);

                // Error recovery
                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
                    handleCreateTableErrorRecovery(stmt);
                }

                // Collect errors
                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
                    copyErrorsFromStatement(stmt);
                }
            } catch (Exception ex) {
                // Use inherited exception handler
                handleStatementParsingException(stmt, i, ex);
                continue;
            }
        }

        // Clean up frame stack
        if (globalFrame != null) globalFrame.popMeFromStack(frameStack);

        return sqlstatements;
    }

    /**
     * Post-processing hook after each statement is parsed.
     * <p>
     * Default implementation does nothing. Override if needed for vendor-specific
     * post-processing.
     *
     * @param stmt the statement that was just parsed
     */
    protected void afterStatementParsed(TCustomSqlStatement stmt) {
        // Default: no post-processing needed for Databricks
    }

    /**
     * Handle error recovery for CREATE TABLE statements.
     * <p>
     * This method attempts to recover from parse errors in CREATE TABLE statements
     * by marking unparseable table properties (like ROW FORMAT, STORED AS, etc.)
     * as sqlpluscmd and retrying.
     * <p>
     * Databricks/Hive DDL allows complex table properties after the column definition
     * that may not be fully supported in the grammar. This error recovery allows
     * partial parsing of the main table structure.
     * <p>
     * Extracted from TGSqlParser.doparse() lines 16916-16971
     *
     * @param stmt the statement with errors
     */
    protected void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
        if (((stmt.sqlstatementtype == ESqlStatementType.sstcreatetable) ||
             (stmt.sqlstatementtype == ESqlStatementType.sstcreateindex)) &&
            (!TBaseType.c_createTableStrictParsing)) {

            // Find the closing parenthesis of table/column definition
            // Mark everything after it as sqlpluscmd (ignored table properties)
            int nested = 0;
            boolean isIgnore = false, isFoundIgnoreToken = false;
            TSourceToken firstIgnoreToken = null;

            for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
                TSourceToken st = stmt.sourcetokenlist.get(k);

                if (isIgnore) {
                    // Mark tokens after closing paren as sqlpluscmd (to be ignored)
                    if (st.issolidtoken() && (st.tokencode != ';')) {
                        isFoundIgnoreToken = true;
                        if (firstIgnoreToken == null) {
                            firstIgnoreToken = st;
                        }
                    }
                    if (st.tokencode != ';') {
                        st.tokencode = TBaseType.sqlpluscmd;
                    }
                    continue;
                }

                // Track nested parentheses to find the matching closing paren
                if (st.tokencode == (int) ')') {
                    nested--;
                    if (nested == 0) {
                        // Check if next token is "AS ( SELECT" - don't ignore CTAS subquery
                        boolean isSelect = false;
                        TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
                        if (st1 != null) {
                            TSourceToken st2 = st.searchToken((int) '(', 2);
                            if (st2 != null) {
                                TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
                                isSelect = (st3 != null);
                            }
                        }
                        if (!isSelect) {
                            // Found the closing paren, start ignoring subsequent tokens
                            isIgnore = true;
                        }
                    }
                }

                if ((st.tokencode == (int) '(') || (st.tokencode == TBaseType.left_parenthesis_2)) {
                    nested++;
                }
            }

            // For Databricks, we don't validate specific table properties
            // (unlike Oracle which checks TBaseType.searchOracleTablePros)
            // This allows any Hive/Databricks DDL syntax like:
            // ROW FORMAT, STORED AS, TBLPROPERTIES, LOCATION, etc.

            // Retry parsing if we found ignoreable properties
            if (isFoundIgnoreToken) {
                stmt.clearError();
                stmt.parsestatement(null, false);
            }
        }
    }

    /**
     * Determine whether a POSITION token starts the SQL-standard
     * {@code POSITION(substr IN str)} form (as opposed to the comma form
     * {@code POSITION(substr, str[, pos])}).
     *
     * <p>Scans the parenthesized argument group that immediately follows the
     * POSITION keyword and returns {@code true} only when the <em>top level</em>
     * of the argument list is an {@code IN} expression — i.e. the first
     * top-level separator encountered is the {@code IN} keyword rather than a
     * comma. "Top level" means paren-depth 1 and outside any {@code CASE ... END}
     * block or {@code [ ... ]} subscript, so an {@code IN} nested inside a
     * sub-expression does not misclassify a comma-form call. This handles:
     * <ul>
     *   <li>{@code POSITION(a, f(x IN y))} — IN inside nested parens (skipped)</li>
     *   <li>{@code POSITION('a', CASE WHEN x IN (..) THEN b END)} — IN inside a
     *       CASE, with a preceding top-level comma (comma form wins)</li>
     *   <li>{@code POSITION(CASE WHEN x IN (..) THEN y END)} — IN inside a CASE,
     *       no top-level IN (comma/generic form)</li>
     *   <li>{@code POSITION(arr[x IN y], str)} — IN inside a subscript (skipped)</li>
     *   <li>{@code POSITION(x::map<string,string>::string IN lower(str))} — the
     *       comma inside the complex-type parameters is not the separator</li>
     * </ul>
     * A quoted identifier named {@code in} is ignored because it lexes as an
     * identifier ({@code ttidentifier}), not the {@code IN} keyword. Angle
     * brackets are only treated as type parameters when the {@code <} follows a
     * complex-type keyword (MAP/ARRAY/STRUCT), so comparison operators such as
     * {@code a < b} are unaffected.
     *
     * @param positionTok the POSITION source token
     * @return {@code true} if the argument list uses the {@code IN} form
     */
    private boolean isPositionInForm(TSourceToken positionTok) {
        TSourceToken open = positionTok.nextSolidToken();
        if ((open == null) || (open.tokencode != (int) '(')) {
            return false;
        }
        int parenDepth = 0;
        int bracketDepth = 0;
        int angleDepth = 0;
        int caseDepth = 0;
        TSourceToken prevSolid = null;
        for (int i = open.posinlist; i < sourcetokenlist.size(); i++) {
            TSourceToken st = sourcetokenlist.get(i);
            if (!st.issolidtoken()) {
                continue;
            }
            if ((st.tokencode == (int) '(') || (st.tokencode == TBaseType.left_parenthesis_2)) {
                parenDepth++;
            } else if (st.tokencode == (int) ')') {
                parenDepth--;
                if (parenDepth == 0) {
                    // reached the matching close paren without a top-level IN
                    return false;
                }
            } else if (st.tokencode == (int) '[') {
                bracketDepth++;
            } else if (st.tokencode == (int) ']') {
                if (bracketDepth > 0) {
                    bracketDepth--;
                }
            } else if ((st.tokentype == ETokenType.ttlessthan) && isComplexTypeKeyword(prevSolid)) {
                // opening a MAP<..>/ARRAY<..>/STRUCT<..> type parameter list
                angleDepth++;
            } else if ((st.tokentype == ETokenType.ttgreaterthan) && (angleDepth > 0)) {
                // nested types lex as separate '>' tokens (e.g. map<..,array<..>>), so
                // each closer decrements one level
                angleDepth--;
            } else if ((parenDepth == 1) && (bracketDepth == 0) && (angleDepth == 0)) {
                if (st.tokentype == ETokenType.ttkeyword) {
                    // non-identifier-compare: matching SQL keywords (CASE/END/IN), not DB object names
                    if (st.toString().equalsIgnoreCase("case")) {
                        caseDepth++;
                    } else if (st.toString().equalsIgnoreCase("end")) {
                        if (caseDepth > 0) {
                            caseDepth--;
                        }
                    } else if ((caseDepth == 0) && st.toString().equalsIgnoreCase("in")) {
                        // top-level IN => POSITION(substr IN str)
                        return true;
                    }
                } else if ((caseDepth == 0) && (st.tokencode == (int) ',')) {
                    // top-level comma => POSITION(substr, str[, pos]); keep generic
                    return false;
                }
            }
            prevSolid = st;
        }
        return false;
    }

    /**
     * @return {@code true} if the token is a complex-type keyword whose type
     *         parameters are written with angle brackets (MAP/ARRAY/STRUCT).
     */
    private static boolean isComplexTypeKeyword(TSourceToken st) {
        if ((st == null) || (st.tokentype != ETokenType.ttkeyword)) {
            return false;
        }
        // non-identifier-compare: matching type keywords, not DB object names
        return st.toString().equalsIgnoreCase("map")
                || st.toString().equalsIgnoreCase("array")
                || st.toString().equalsIgnoreCase("struct");
    }

    // ========== Semantic Analysis and Interpretation ==========

    /**
     * Perform semantic analysis (resolve column-table relationships, etc.).
     * <p>
     * This method runs the TSQLResolver to build semantic relationships
     * between columns and tables, among other analysis.
     *
     * @param context parser context
     * @param statements statement list to analyze
     */
    @Override
    protected void performSemanticAnalysis(ParserContext context, TStatementList statements) {
        if (!TBaseType.isEnableResolver()) {
            return;
        }

        if (!getSyntaxErrors().isEmpty()) {
            return;
        }

        try {
            TSQLResolver resolver = new TSQLResolver(globalContext, statements);
            resolver.resolve();
        } catch (Exception e) {
            // Log but don't fail - semantic analysis is optional
            System.err.println("Semantic analysis failed: " + e.getMessage());
        }
    }

    /**
     * Perform interpretation (execute SQL in interpreter mode).
     * <p>
     * This method runs the TASTEvaluator to interpret/execute the SQL.
     *
     * @param context parser context
     * @param statements statement list to interpret
     */
    @Override
    protected void performInterpreter(ParserContext context, TStatementList statements) {
        if (!TBaseType.ENABLE_INTERPRETER) {
            return;
        }

        try {
            TGlobalScope interpreterScope = new TGlobalScope(sqlEnv);
            TLog.enableInterpreterLogOnly();
            TASTEvaluator astEvaluator = new TASTEvaluator(statements, interpreterScope);
            astEvaluator.eval();
        } catch (Exception e) {
            // Log but don't fail - interpretation is optional
            System.err.println("Interpretation failed: " + e.getMessage());
        }
    }

    @Override
    public String toString() {
        return "DatabricksSqlParser{vendor=" + vendor + "}";
    }
}