package gudusoft.gsqlparser.parser;

import gudusoft.gsqlparser.EDbVendor;
import gudusoft.gsqlparser.EErrorType;
import gudusoft.gsqlparser.ESqlStatementType;
import gudusoft.gsqlparser.ETokenStatus;
import gudusoft.gsqlparser.ETokenType;
import gudusoft.gsqlparser.TrialInputGuard;
import gudusoft.gsqlparser.TBaseType;
import gudusoft.gsqlparser.TCustomLexer;
import gudusoft.gsqlparser.TCustomParser;
import gudusoft.gsqlparser.TCustomSqlStatement;
import gudusoft.gsqlparser.TSourceToken;
import gudusoft.gsqlparser.TSourceTokenList;
import gudusoft.gsqlparser.TStatementList;
import gudusoft.gsqlparser.TSyntaxError;
import gudusoft.gsqlparser.compiler.TContext;
import gudusoft.gsqlparser.compiler.TFrame;
import gudusoft.gsqlparser.compiler.TGlobalScope;
import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
import gudusoft.gsqlparser.sqlenv.TSQLEnv;
import gudusoft.gsqlparser.stmt.TRoutine;

import java.io.ByteArrayInputStream;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

/**
 * Abstract base class providing common logic and template methods for SQL parsing.
 *
 * <p>This class implements the <b>Template Method Pattern</b>, defining the skeleton
 * of the parsing algorithm while allowing subclasses to override specific steps.
 * It provides default implementations for common operations and hooks for
 * vendor-specific customization.
 *
 * <p><b>Design Pattern:</b> Template Method
 * <ul>
 *   <li><b>Template Methods:</b> {@link #parse(ParserContext)}, {@link #tokenize(ParserContext)}</li>
 *   <li><b>Abstract Methods:</b> Must be implemented by subclasses</li>
 *   <li><b>Hook Methods:</b> Optional overrides for customization</li>
 * </ul>
 *
 * <p><b>Parsing Algorithm (Template Method):</b>
 * <ol>
 *   <li>Get lexer ({@link #getLexer(ParserContext)})</li>
 *   <li>Tokenize SQL ({@link #performTokenization(ParserContext, TCustomLexer)})</li>
 *   <li>Process tokens ({@link #processTokensBeforeParse(ParserContext, TSourceTokenList)})</li>
 *   <li>Get parser(s) ({@link #getParser(ParserContext, TSourceTokenList)})</li>
 *   <li>Parse SQL ({@link #performParsing(ParserContext, TCustomParser, TCustomParser, TSourceTokenList)})</li>
 *   <li>Semantic analysis ({@link #performSemanticAnalysis(ParserContext, TStatementList)})</li>
 * </ol>
 *
 * <p><b>Subclass Responsibilities:</b>
 * <pre>
 * public class OracleSqlParser extends AbstractSqlParser {
 *     public OracleSqlParser() {
 *         super(EDbVendor.dbvoracle);
 *         this.delimiterChar = '/';
 *     }
 *
 *     // Must implement abstract methods
 *     protected TCustomLexer getLexer(ParserContext context) {
 *         return new TLexerOracle();
 *     }
 *
 *     protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
 *         return new TParserOracleSql(tokens);
 *     }
 *
 *     // ... other abstract methods
 *
 *     // Optionally override hook methods
 *     protected void processTokensBeforeParse(ParserContext context, TSourceTokenList tokens) {
 *         // Oracle-specific token processing
 *     }
 * }
 * </pre>
 *
 * @see SqlParser
 * @see ParserContext
 * @see SqlParseResult
 * @since 3.2.0.0
 */
public abstract class AbstractSqlParser implements SqlParser {

    protected final EDbVendor vendor;
    protected char delimiterChar = ';';
    protected String defaultDelimiterStr = ";";

    // Syntax errors collected during parsing
    protected List<TSyntaxError> syntaxErrors = new ArrayList<>();

    // ========== Core Parsing Components (Reused Across Parse Operations) ==========

    /**
     * Token list container - created once in constructor, cleared before each parse.
     * <p>This follows the component reuse pattern to avoid allocation overhead.
     */
    protected TSourceTokenList sourcetokenlist;

    /**
     * Statement list container - created once in constructor, cleared before each extraction.
     * <p>This follows the component reuse pattern to avoid allocation overhead.
     */
    protected TStatementList sqlstatements;

    /**
     * Current parser context for the ongoing parse operation.
     * <p>Set at the beginning of each parse operation, contains input SQL and options.
     */
    protected ParserContext parserContext;

    /**
     * SQL command resolver for identifying statement types (SELECT, INSERT, etc.).
     * <p>Initialized lazily using SqlCmdsFactory.get(vendor) - vendor-specific implementation.
     */
    protected ISqlCmds sqlcmds;

    /**
     * Token handler callback for processing tokens as they are created.
     * <p>Optional callback that gets invoked for each token created during tokenization.
     */
    private gudusoft.gsqlparser.ITokenHandle tokenHandle = null;

    /**
     * The lexer instance used for tokenization.
     * <p>Subclasses should set this field in their constructor to their specific lexer instance.
     * This allows common tokenization logic in AbstractSqlParser to access the lexer generically.
     */
    protected TCustomLexer lexer = null;

    // ========== Semantic Analysis Infrastructure ==========

    /**
     * Global context for semantic analysis.
     * <p>Created during performParsing phase, contains SQL environment and statement references.
     */
    protected TContext globalContext;

    /**
     * SQL environment for semantic analysis.
     * <p>Vendor-specific environment configuration, used by resolver and semantic analysis.
     */
    protected TSQLEnv sqlEnv;

    /**
     * Frame stack for scope management during parsing.
     * <p>Used to track nested scopes (global, statement, block-level) during parsing.
     */
    protected java.util.Stack<TFrame> frameStack;

    /**
     * Global frame pushed to frame stack during parsing.
     * <p>Represents the outermost scope, must be popped after parsing completes.
     */
    protected TFrame globalFrame;

    protected static class PreparedSqlReader {
        private final BufferedReader reader;
        private final String charset;

        protected PreparedSqlReader(BufferedReader reader, String charset) {
            this.reader = reader;
            this.charset = charset;
        }

        public BufferedReader getReader() {
            return reader;
        }

        public String getCharset() {
            return charset;
        }
    }

    /**
     * Construct parser for given database vendor.
     *
     * @param vendor the database vendor
     */
    protected AbstractSqlParser(EDbVendor vendor) {
        if (vendor == null) {
            throw new IllegalArgumentException("vendor cannot be null");
        }
        this.vendor = vendor;

        // Initialize reusable containers (cleared before each use)
        this.sourcetokenlist = new TSourceTokenList();
        this.sqlstatements = new TStatementList();

        // Note: parserContext is set at the beginning of each parse operation
        // Note: sqlcmds is initialized lazily when first needed
    }

    @Override
    public EDbVendor getVendor() {
        return vendor;
    }

    /**
     * Set an event handler which will be fired when a new source token is created by the lexer during tokenization.
     *
     * @param tokenHandle the event handler to process the new created source token
     */
    public void setTokenHandle(gudusoft.gsqlparser.ITokenHandle tokenHandle) {
        this.tokenHandle = tokenHandle;
    }

    /**
     * Template method for full parsing.
     *
     * <p>This method defines the skeleton of the parsing algorithm.
     * Subclasses should NOT override this method; instead, they should
     * override the abstract methods and hook methods called by this template.
     *
     * <p><b>Algorithm:</b>
     * <ol>
     *   <li>Create lexer</li>
     *   <li>Tokenize (time tracked)</li>
     *   <li>Process tokens (vendor-specific preprocessing)</li>
     *   <li>Create parser(s)</li>
     *   <li>Parse (time tracked)</li>
     *   <li>Semantic analysis (time tracked)</li>
     *   <li>Interpreter (time tracked)</li>
     * </ol>
     *
     * @param context immutable context with all inputs
     * @return immutable result with all outputs
     */
    @Override
    public final SqlParseResult parse(ParserContext context) {
        // Clear syntax errors from previous parse
        syntaxErrors.clear();

        try {
            // Step 1: Get raw statements (internally calls tokenize() and extractRawStatements())
            SqlParseResult rawResult = getrawsqlstatements(context);

            if (rawResult.getErrorCode() != 0) {
                return rawResult;
            }

            // Get tokens, lexer, and RAW STATEMENTS from raw result
            TSourceTokenList tokens = rawResult.getSourceTokenList();
            TCustomLexer lexer = rawResult.getLexer();
            TStatementList rawStatements = rawResult.getSqlStatements();

            // Step 2: Get parser(s)
            TCustomParser parser = getParser(context, tokens);
            TCustomParser secondaryParser = getSecondaryParser(context, tokens);

            // Step 3: Full parsing (build AST for each raw statement)
            SqlParseResult.Builder resultBuilder = new SqlParseResult.Builder();
            resultBuilder.lexer(lexer);
            resultBuilder.sourceTokenList(tokens);
            resultBuilder.tokenizationTimeMs(rawResult.getTokenizationTimeMs());
            resultBuilder.parser(parser);

            String initialCatalog = sqlEnv == null ? null : sqlEnv.getDefaultCatalogName();
            String initialSchema = sqlEnv == null ? null : sqlEnv.getDefaultSchemaName();
            long parseStart = System.currentTimeMillis();
            // Pass raw statements to performParsing - it will build AST for each statement
            TStatementList statements = performParsing(context, parser, secondaryParser, tokens, rawStatements);
            if (statements == null) {
                statements = new TStatementList();
            }
            resultBuilder.sqlStatements(statements);
            resultBuilder.parsingTimeMs(System.currentTimeMillis() - parseStart);

            // Step 4: Semantic analysis
            if (!context.isOnlyNeedRawParseTree()) {
                long semanticStart = System.currentTimeMillis();
                String parsedCatalog = sqlEnv == null ? null : sqlEnv.getDefaultCatalogName();
                String parsedSchema = sqlEnv == null ? null : sqlEnv.getDefaultSchemaName();
                boolean replayDefaults = TBaseType.isEnableResolver2() && sqlEnv != null;
                if (replayDefaults) {
                    sqlEnv.setDefaultCatalogName(initialCatalog);
                    sqlEnv.setDefaultSchemaName(initialSchema);
                }
                try {
                    performSemanticAnalysis(context, statements);
                } finally {
                    if (replayDefaults) {
                        sqlEnv.setDefaultCatalogName(parsedCatalog);
                        sqlEnv.setDefaultSchemaName(parsedSchema);
                    }
                }
                resultBuilder.semanticAnalysisTimeMs(System.currentTimeMillis() - semanticStart);
            }

            // Step 5: Interpreter
            if (!context.isOnlyNeedRawParseTree() && syntaxErrors.isEmpty()) {
                long interpreterStart = System.currentTimeMillis();
                performInterpreter(context, statements);
                resultBuilder.interpreterTimeMs(System.currentTimeMillis() - interpreterStart);
            }

            resultBuilder.syntaxErrors(syntaxErrors instanceof ArrayList ?
                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
            resultBuilder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size());
            resultBuilder.errorMessage(syntaxErrors.isEmpty() ? "" :
                String.format("Parsing completed with %d error(s)", syntaxErrors.size()));

            return resultBuilder.build();
        } catch (Exception e) {
            e.printStackTrace();
            SqlParseResult.Builder resultBuilder = new SqlParseResult.Builder();
            resultBuilder.errorCode(1);
            String errorMsg = "Parsing failed: " + e.getMessage();
            resultBuilder.errorMessage(errorMsg);
            System.out.println(errorMsg+"File:\t"+context.getSqlFilename());
            if (context.isDumpResolverLog()) {
                e.printStackTrace();
            }
            return resultBuilder.build();
        }
    }

    /**
     * Template method for tokenization only (without full parsing).
     *
     * <p>This method is used by {@code getrawsqlstatements()} which only
     * needs tokenization and raw statement extraction, without detailed
     * syntax checking or semantic analysis.
     *
     * <p><b>Algorithm:</b>
     * <ol>
     *   <li>Get lexer</li>
     *   <li>Tokenize (time tracked)</li>
     *   <li>Extract raw statements (no parsing)</li>
     * </ol>
     *
     * @param context immutable context with all inputs
     * @return immutable result with tokens and raw statements
     */
    @Override
    public final SqlParseResult tokenize(ParserContext context) {
        SqlParseResult.Builder resultBuilder = new SqlParseResult.Builder();

        try {
            // Step 1: Get lexer (vendor-specific instance, may be cached)
            TCustomLexer lexer = getLexer(context);
            if (lexer == null) {
                throw new IllegalStateException("getLexer() returned null");
            }
            resultBuilder.lexer(lexer);

            // Step 2: Perform tokenization. prepareSqlReader() measures the
            // input at the byte source and records a verdict BEFORE any token
            // is produced, so a refused input never yields tokens at all.
            long tokenStart = System.currentTimeMillis();
            trialRefusalMessage = null;
            TSourceTokenList tokens = performTokenization(context, lexer);
            if (tokens == null) {
                throw new IllegalStateException("performTokenization() returned null");
            }

            // Trial size verdict, recorded at the byte source by prepareSqlReader
            // before a single character was decoded. Reaching here with a
            // refusal means tokenization ran over an intentionally empty reader,
            // so there is nothing to retract -- but clear anyway and report the
            // canonical message. Always null in full builds.
            if (trialRefusalMessage != null) {
                String refusal = trialRefusalMessage;
                trialRefusalMessage = null;
                tokens.clear();
                syntaxErrors.clear();
                resultBuilder.errorCode(-1);
                resultBuilder.errorMessage(refusal);
                return resultBuilder.build();
            }


            // Step 3: Post-tokenization processing (CRITICAL for correct behavior)
            // These steps must run after tokenization to prepare tokens for parsing

            // Step 3a: Post-tokenization normalization
            doAfterTokenize(tokens);

            // Step 3b: Reset token chain (CRITICAL FIX)
            // Links all tokens via getNextTokenInChain() - required for TObjectName.toString()
            TBaseType.resetTokenChain(tokens, 0);

            // Step 3c: Process tokens using token table
            // Vendor-specific token code adjustments (e.g., BigQuery/Snowflake DO keyword handling)
            processTokensInTokenTable(context, lexer, tokens);

            // Step 3d: Pre-parse token processing
            // Pre-parse preprocessing (e.g., Snowflake duplicate semicolon removal)
            processTokensBeforeParse(context, tokens);

            // Step 3e: Invoke the public token-list hook after all built-in
            // normalization, matching the legacy TGSqlParser tokenization path.
            if (context.getTokenListHandle() != null) {
                context.getTokenListHandle().processTokenList(tokens);
                // A handler may delete or merge tokens. Rebuild the linked token
                // chain before raw statement extraction and parsing consume it.
                TBaseType.resetTokenChain(tokens, 0);
            }

            resultBuilder.sourceTokenList(tokens);
            resultBuilder.tokenizationTimeMs(System.currentTimeMillis() - tokenStart);

            // Success
            resultBuilder.errorCode(0);
            resultBuilder.errorMessage("");

        } catch (Exception e) {
            // Error occurred
            resultBuilder.errorCode(1);
            String errorMsg = "Tokenization failed: " + e.getMessage();
            resultBuilder.errorMessage(errorMsg);

            // Log error if enabled
            if (context.isDumpResolverLog()) {
                e.printStackTrace();
            }
        }

        return resultBuilder.build();
    }

    /**
     * Template method for extracting raw statements without full parsing.
     *
     * <p>This method performs tokenization and raw statement extraction,
     * but skips the expensive full parsing and semantic analysis steps.
     *
     * <p><b>Algorithm:</b>
     * <ol>
     *   <li>Tokenize SQL (via {@link #tokenize(ParserContext)})</li>
     *   <li>Extract raw statements (via {@link #extractRawStatements(ParserContext, TSourceTokenList, TCustomLexer, long)})</li>
     *   <li>Return result with tokens and raw statements</li>
     * </ol>
     *
     * <p><b>Equivalent to legacy API:</b> {@code TGSqlParser.getrawsqlstatements()}
     *
     * @param context immutable context with all inputs
     * @return immutable result with tokens and raw statements (no AST)
     */
    @Override
    public final SqlParseResult getrawsqlstatements(ParserContext context) {
        try {
            // Step 1: Tokenize with all post-processing (calls tokenize())
            SqlParseResult tokenizeResult = tokenize(context);

            // Check tokenization result
            if (tokenizeResult.getErrorCode() != 0) {
                return tokenizeResult;
            }

            // Get tokens and lexer from tokenize result
            TSourceTokenList tokens = tokenizeResult.getSourceTokenList();
            TCustomLexer lexer = tokenizeResult.getLexer();
            long tokenizationTimeMs = tokenizeResult.getTokenizationTimeMs();

            // Step 2: Extract raw statements (vendor-specific)
            // Vendor implementation creates builder, populates it, and returns complete result
            SqlParseResult extractResult = extractRawStatements(context, tokens, lexer, tokenizationTimeMs);

            return extractResult;
        } catch (Exception e) {
            e.printStackTrace();
            SqlParseResult.Builder resultBuilder = new SqlParseResult.Builder();
            resultBuilder.errorCode(1);
            resultBuilder.errorMessage("Raw statement extraction failed: " + e.getMessage() );
            if (context.isDumpResolverLog()) {
                e.printStackTrace();
            }
            return resultBuilder.build();
        }
    }



    // ========== Abstract Methods (MUST be implemented by subclasses) ==========

    /**
     * Get the lexer for this vendor.
     *
     * <p><b>Subclass Responsibility:</b> Return vendor-specific lexer instance.
     * The lexer may be created fresh or cached/reused for performance.
     *
     * <p><b>Example:</b>
     * <pre>
     * protected TCustomLexer getLexer(ParserContext context) {
     *     TLexerOracle lexer = new TLexerOracle();
     *     lexer.delimiterchar = delimiterChar;
     *     lexer.defaultDelimiterStr = defaultDelimiterStr;
     *     return lexer;
     * }
     * </pre>
     *
     * @param context the parser context
     * @return configured lexer instance (never null)
     */
    protected abstract TCustomLexer getLexer(ParserContext context);

    /**
     * Get the main parser for this vendor.
     *
     * <p><b>Subclass Responsibility:</b> Return vendor-specific parser instance.
     * The parser may be created fresh or cached/reused for performance.
     * If reusing, the token list should be updated.
     *
     * <p><b>Example:</b>
     * <pre>
     * protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
     *     TParserOracleSql parser = new TParserOracleSql(tokens);
     *     parser.lexer = getLexer(context);
     *     return parser;
     * }
     * </pre>
     *
     * @param context the parser context
     * @param tokens the source token list
     * @return configured parser instance (never null)
     */
    protected abstract TCustomParser getParser(ParserContext context, TSourceTokenList tokens);

    /**
     * Perform tokenization using vendor-specific lexer.
     *
     * <p><b>Template Method:</b> This method implements the common tokenization
     * algorithm across all database vendors. Subclasses customize through one hook:
     * {@link #tokenizeVendorSql()} - Call vendor-specific tokenization logic
     *
     * <p><b>Algorithm:</b>
     * <ol>
     *   <li>Store parser context</li>
     *   <li>Prepare SQL reader (file/string with charset detection)</li>
     *   <li>Configure lexer with input reader and charset</li>
     *   <li>Reset lexer state</li>
     *   <li>Clear token list and reset position</li>
     *   <li>Reset token table cache</li>
     *   <li>Call {@link #tokenizeVendorSql()} hook</li>
     *   <li>Return populated token list</li>
     * </ol>
     *
     * @param context parser context with SQL input configuration
     * @param lexer the lexer instance (same as this.flexer)
     * @return token list populated by vendor-specific tokenization
     * @throws RuntimeException if tokenization fails
     */
    protected TSourceTokenList performTokenization(ParserContext context, TCustomLexer lexer) {
        this.parserContext = context;

        // Set token handle from context if provided (allows TGSqlParser.setTokenHandle() to work)
        if (context.getTokenHandle() != null) {
            this.tokenHandle = context.getTokenHandle();
        }

        try {
            PreparedSqlReader prepared = prepareSqlReader(context);
            BufferedReader finputstream = prepared.getReader();
            String effectiveCharset = prepared.getCharset();

            // Configure lexer with input (lexer is vendor-specific flexer from subclass)
            lexer.yyinput = finputstream;
            if (effectiveCharset != null && !effectiveCharset.isEmpty()) {
                lexer.setSqlCharset(effectiveCharset);
            }
            lexer.reset();

            // Reset token list
            this.sourcetokenlist.clear();
            this.sourcetokenlist.curpos = -1;

            // Reset token table cache
            lexer.resetTokenTable();

            // HOOK: Call vendor-specific tokenization
            try {
                tokenizeVendorSql();
            } finally {
                // Tokenization has fully drained the input into sourcetokenlist;
                // nothing reads yyinput after this point. Closing here releases
                // the file descriptor the reader may hold -- without it, every
                // file parse on this path leaked one descriptor for the life of
                // the JVM. A caller-supplied stream is closed too: it is at EOF
                // and the legacy path (TGSqlParser.readsql) closes it the same
                // way on the next parse.
                try {
                    finputstream.close();
                } catch (IOException ignored) {
                    // Nothing to do: the input was already consumed.
                }
            }

            return this.sourcetokenlist;

        } catch (Exception e) {
            throw new RuntimeException("Tokenization failed: " + e.getMessage(), e);
        }
    }

    /**
     * Call vendor-specific tokenization logic.
     *
     * <p><b>Hook Method:</b> Called by {@link #performTokenization} to execute
     * vendor-specific SQL-to-token conversion logic.
     *
     * <p><b>Subclass Responsibility:</b> Call the vendor-specific tokenization method
     * (e.g., dooraclesqltexttotokenlist, domssqlsqltexttotokenlist) which reads
     * from lexer and populates sourcetokenlist.
     *
     * <p><b>Example (Oracle):</b>
     * <pre>
     * protected void tokenizeVendorSql() {
     *     dooraclesqltexttotokenlist();
     * }
     * </pre>
     *
     * <p><b>Example (MSSQL):</b>
     * <pre>
     * protected void tokenizeVendorSql() {
     *     domssqlsqltexttotokenlist();
     * }
     * </pre>
     *
     * <p><b>Example (PostgreSQL):</b>
     * <pre>
     * protected void tokenizeVendorSql() {
     *     dopostgresqltexttotokenlist();
     * }
     * </pre>
     */
    protected abstract void tokenizeVendorSql();

    /**
     * Extract raw statements without full parsing (public API).
     *
     * <p>This public method allows external callers (like TGSqlParser) to extract
     * raw statements from an already-tokenized source list without re-tokenization.
     *
     * @param context the parser context
     * @param tokens the source token list (already tokenized)
     * @return statement list (never null)
     * @since 3.2.0.0
     */
    public final TStatementList doExtractRawStatements(ParserContext context, TSourceTokenList tokens) {
        // Caller-supplied tokens never passed a byte source, so there is nothing
        // to measure at ingestion: a caller can build a list by hand, or widen a
        // token from an accepted small parse past the cap. Measure the token
        // text itself, joined and encoded ONCE. No-op in full builds.
        if (TrialInputGuard.tokenTextExceedsCap(tokens)) {
            return new TStatementList();
        }

        // Create a dummy lexer since we already have tokens
        TCustomLexer lexer = getLexer(context);

        // Call vendor-specific extraction and extract statement list from result
        SqlParseResult result = extractRawStatements(context, tokens, lexer, 0);
        return result.getSqlStatements() != null ? result.getSqlStatements() : new TStatementList();
    }


    /**
     * Extract raw statements without full parsing.
     *
     * <p><b>Template Method:</b> This method implements the common algorithm for
     * extracting raw statements across all database vendors. Subclasses customize
     * the process through two hook methods:
     * <ul>
     *   <li>{@link #setupVendorParsersForExtraction()} - Initialize vendor parsers</li>
     *   <li>{@link #extractVendorRawStatements(SqlParseResult.Builder)} - Call vendor extraction logic</li>
     * </ul>
     *
     * <p><b>Algorithm:</b>
     * <ol>
     *   <li>Create SqlParseResult.Builder</li>
     *   <li>Set common fields (lexer, tokens, tokenization time)</li>
     *   <li>Store context and tokens for extraction</li>
     *   <li>Initialize SQL command resolver</li>
     *   <li>Call {@link #setupVendorParsersForExtraction()} hook</li>
     *   <li>Time the extraction</li>
     *   <li>Call {@link #extractVendorRawStatements(SqlParseResult.Builder)} hook</li>
     *   <li>Set parsing time</li>
     *   <li>Build and return result</li>
     * </ol>
     *
     * @param context the parser context
     * @param tokens the source token list
     * @param lexer the lexer instance (for including in result)
     * @param tokenizationTimeMs tokenization time from tokenize() step
     * @return complete SqlParseResult with raw statements and metadata
     */
    protected SqlParseResult extractRawStatements(ParserContext context,
                                                  TSourceTokenList tokens,
                                                  TCustomLexer lexer,
                                                  long tokenizationTimeMs) {
        // Create builder for result construction
        SqlParseResult.Builder builder = new SqlParseResult.Builder();

        // Set common result fields
        builder.lexer(lexer);
        builder.sourceTokenList(tokens);
        builder.tokenizationTimeMs(tokenizationTimeMs);

        // CRITICAL: Include parser(s) in result so TGSqlParser can use them in common parsing loop
        TCustomParser parser = getParser(context, tokens);
        builder.parser(parser);

        // Include secondary parser for vendors that have one (e.g., Oracle PL/SQL parser)
        TCustomParser secondaryParser = getSecondaryParser(context, tokens);
        if (secondaryParser != null) {
            builder.secondaryParser(secondaryParser);
        }

        // Store context and tokens for extraction
        this.sourcetokenlist = tokens;
        if (this.sqlstatements == null) {
            this.sqlstatements = new TStatementList();
        } else {
            this.sqlstatements.clear();
        }
        this.syntaxErrors.clear();  // Clear syntax errors from previous extraction
        this.parserContext = context;

        // Initialize SQL command resolver (if not already done). Use the
        // context vendor: a delegated parser's own vendor is the grammar
        // family (e.g. MssqlSqlParser is dbvmssql even for dbvexasol input),
        // while statement recognition must follow the requested dialect.
        if (this.sqlcmds == null) {
            this.sqlcmds = SqlCmdsFactory.get(context != null ? context.getVendor() : vendor);
        }

        // HOOK 1: Vendor-specific parser setup (sqlcmds injection, token list update)
        setupVendorParsersForExtraction();

        // Time the extraction
        long extractStart = System.currentTimeMillis();

        // HOOK 2: Call vendor-specific raw statement extraction
        extractVendorRawStatements(builder);

        builder.parsingTimeMs(System.currentTimeMillis() - extractStart);

        // Add extracted statements to result
        builder.sqlStatements(this.sqlstatements);

        // Add collected syntax errors to result
        if (!syntaxErrors.isEmpty()) {
            builder.syntaxErrors(syntaxErrors instanceof ArrayList ?
                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
        }

        return builder.build();
    }

    /**
     * Setup vendor-specific parsers for raw statement extraction.
     *
     * <p><b>Hook Method:</b> Called by {@link #extractRawStatements} after initializing
     * sqlcmds but before calling the vendor-specific extraction logic.
     *
     * <p><b>Subclass Responsibility:</b> Inject sqlcmds into vendor parser(s) and
     * update their token lists. Examples:
     * <ul>
     *   <li><b>Single parser (MSSQL):</b> Inject into fparser only</li>
     *   <li><b>Dual parsers (Oracle):</b> Inject into both fparser and fplsqlparser</li>
     * </ul>
     *
     * <p><b>Example (MSSQL):</b>
     * <pre>
     * protected void setupVendorParsersForExtraction() {
     *     this.fparser.sqlcmds = this.sqlcmds;
     *     this.fparser.sourcetokenlist = this.sourcetokenlist;
     * }
     * </pre>
     *
     * <p><b>Example (Oracle with dual parsers):</b>
     * <pre>
     * protected void setupVendorParsersForExtraction() {
     *     this.fparser.sqlcmds = this.sqlcmds;
     *     this.fplsqlparser.sqlcmds = this.sqlcmds;
     *     this.fparser.sourcetokenlist = this.sourcetokenlist;
     *     this.fplsqlparser.sourcetokenlist = this.sourcetokenlist;
     * }
     * </pre>
     */
    protected abstract void setupVendorParsersForExtraction();

    /**
     * Call vendor-specific raw statement extraction logic.
     *
     * <p><b>Hook Method:</b> Called by {@link #extractRawStatements} to execute
     * the vendor-specific logic for identifying statement boundaries.
     *
     * <p><b>Subclass Responsibility:</b> Call the vendor-specific extraction method
     * (e.g., dooraclegetrawsqlstatements, domssqlgetrawsqlstatements) passing the
     * builder. The extraction method will populate the builder with raw statements.
     *
     * <p><b>Example (Oracle):</b>
     * <pre>
     * protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
     *     dooraclegetrawsqlstatements(builder);
     * }
     * </pre>
     *
     * <p><b>Example (MSSQL):</b>
     * <pre>
     * protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
     *     domssqlgetrawsqlstatements(builder);
     * }
     * </pre>
     *
     * @param builder the result builder to populate with raw statements
     */
    protected abstract void extractVendorRawStatements(SqlParseResult.Builder builder);

    /**
     * Perform actual parsing with syntax checking.
     *
     * <p><b>Subclass Responsibility:</b> Parse SQL using vendor-specific parser
     * and optional secondary parser (e.g., PL/SQL for Oracle).
     *
     * <p><b>Important:</b> This method receives raw statements that have already been
     * extracted by {@link #getrawsqlstatements(ParserContext)}. Subclasses should NOT
     * re-extract statements - just parse each statement to build the AST.
     *
     * <p><b>Example:</b>
     * <pre>
     * protected TStatementList performParsing(ParserContext context,
     *                                         TCustomParser parser,
     *                                         TCustomParser secondaryParser,
     *                                         TSourceTokenList tokens,
     *                                         TStatementList rawStatements) {
     *     // Use the passed-in rawStatements (DO NOT re-extract!)
     *     for (int i = 0; i &lt; rawStatements.size(); i++) {
     *         TCustomSqlStatement stmt = rawStatements.get(i);
     *         stmt.parsestatement(...);  // Build AST for each statement
     *     }
     *     return rawStatements;
     * }
     * </pre>
     *
     * @param context the parser context
     * @param parser the main parser instance
     * @param secondaryParser secondary parser (may be null)
     * @param tokens the source token list
     * @param rawStatements raw statements already extracted (never null)
     * @return statement list with parsed AST (never null)
     */
    protected abstract TStatementList performParsing(ParserContext context,
                                                     TCustomParser parser,
                                                     TCustomParser secondaryParser,
                                                     TSourceTokenList tokens,
                                                     TStatementList rawStatements);

    // ========== Hook Methods (MAY be overridden by subclasses) ==========

    /**
     * Get secondary parser (e.g., PL/SQL for Oracle).
     *
     * <p><b>Hook Method:</b> Default implementation returns null.
     * Override if vendor needs a secondary parser.
     * The parser may be created fresh or cached/reused for performance.
     *
     * <p><b>Example (Oracle):</b>
     * <pre>
     * protected TCustomParser getSecondaryParser(ParserContext context, TSourceTokenList tokens) {
     *     TParserOraclePLSql plsqlParser = new TParserOraclePLSql(tokens);
     *     plsqlParser.lexer = getLexer(context);
     *     return plsqlParser;
     * }
     * </pre>
     *
     * @param context the parser context
     * @param tokens the source token list
     * @return secondary parser instance, or null if not needed
     */
    protected TCustomParser getSecondaryParser(ParserContext context, TSourceTokenList tokens) {
        return null; // Most vendors don't need this
    }

    /**
     * Post-tokenization normalization.
     * <p>
     * Handles matching parentheses wrapping around SQL and marks semicolons
     * before closing parens to be ignored.
     * <p>
     * Extracted from: TGSqlParser.doAfterTokenize() (lines 5123-5161)
     *
     * @param tokens the source token list (mutable)
     */
    protected void doAfterTokenize(TSourceTokenList tokens) {
        int leftParenCount = 0;
        int rightParenCount = 0;
        int leftIndex = 0;
        int rightIndex = tokens.size() - 1;

        // Count opening parentheses at the beginning
        while (leftIndex < tokens.size() && tokens.get(leftIndex).tokencode == '(') {
            leftParenCount++;
            leftIndex++;
        }

        // Count closing parentheses at the end
        while (rightIndex >= 0 && tokens.get(rightIndex).tokencode == ')') {
            rightParenCount++;
            rightIndex--;
        }

        // Set matching parentheses to be ignored
        int parensToIgnore = Math.min(leftParenCount, rightParenCount);
        // if there is a semicolon before the right parenthesis, set the semicolon to be ignored
        // mantisbt/view.php?id=3690

        if ((parensToIgnore > 0) && (tokens.get(tokens.size() - 1 - (parensToIgnore - 1) - 1).tokencode == ';')){
            // set to whitespace that this semicolon will be ignored during getting raw sql
            tokens.get(tokens.size() - 1 - (parensToIgnore - 1) - 1).tokentype = ETokenType.ttwhitespace;
            // set to ignore by yacc that this semicolon will be ignored during parsing
            tokens.get(tokens.size() - 1 - (parensToIgnore - 1) - 1).tokenstatus = ETokenStatus.tsignorebyyacc;
        }
    }

    /**
     * Process tokens using token table (vendor-specific token code adjustments).
     * <p>
     * Currently handles BigQuery and Snowflake to convert DO keywords to identifiers
     * when there's no corresponding WHILE/FOR.
     * <p>
     * Extracted from: TGSqlParser.processTokensInTokenTable() (lines 5186-5209)
     *
     * @param context the parser context
     * @param lexer the lexer (for accessing TOKEN_TABLE)
     * @param tokens the source token list (mutable)
     */
    protected void processTokensInTokenTable(ParserContext context, TCustomLexer lexer, TSourceTokenList tokens) {
        // Get token table from lexer
        long[][] TOKEN_TABLE1 = lexer.TOKEN_TABLE;

        switch (vendor){
            case dbvbigquery:
            case dbvsnowflake:
                // case 1, DO keyword: if no corresponding FOR, WHILE etc keywords found,
                // set DO keyword's token code to TBaseType.ident
                if (TOKEN_TABLE1[TBaseType.rrw_do][0] > 0){
                    if ((TOKEN_TABLE1[TBaseType.rrw_while][0] == 0) && (TOKEN_TABLE1[TBaseType.rrw_for][0] == 0)){
                        for(int i=0; i<tokens.size(); i++){
                            TSourceToken st = tokens.get(i);
                            if (st.tokencode == TBaseType.rrw_do){
                                st.tokencode = TBaseType.ident;
                            }
                        }
                    }
                }
                break;
        }
    }

    /**
     * Process tokens before parsing (vendor-specific adjustments).
     *
     * <p><b>Hook Method:</b> Default implementation handles Snowflake consecutive semicolons.
     * Override if vendor needs additional token preprocessing.
     *
     * <p>Extracted from: TGSqlParser.processTokensBeforeParse() (lines 5165-5184)
     *
     * <p><b>Example:</b>
     * <pre>
     * protected void processTokensBeforeParse(ParserContext context, TSourceTokenList tokens) {
     *     super.processTokensBeforeParse(context, tokens); // Call base implementation
     *     // Add vendor-specific processing...
     * }
     * </pre>
     *
     * @param context the parser context
     * @param tokens the source token list (mutable)
     */
    protected void processTokensBeforeParse(ParserContext context, TSourceTokenList tokens) {
        // For performance, only process for Snowflake as this is currently only needed there
        // mantisbt/view.php?id=3579
        if (vendor != EDbVendor.dbvsnowflake) return;

        // If there are consecutive semicolon tokens, mark the second semicolon token as deleted
        for(int i=0; i<tokens.size(); i++){
            TSourceToken st = tokens.get(i);
            if (st.tokencode == ';'){
                TSourceToken nextToken = st.nextSolidToken();
                if (nextToken != null){
                    if (nextToken.tokencode == ';'){
                        nextToken.tokenstatus = ETokenStatus.tsdeleted;
                    }
                }
            }
        }
    }

    /**
     * Perform semantic analysis on parsed statements.
     *
     * <p><b>Hook Method:</b> Default implementation does nothing.
     * Override to provide vendor-specific semantic analysis.
     *
     * <p><b>Typical Implementation:</b>
     * <ul>
     *   <li>Column-to-table resolution (TSQLResolver)</li>
     *   <li>Dataflow analysis</li>
     *   <li>Reference resolution</li>
     *   <li>Scope resolution</li>
     * </ul>
     *
     * @param context the parser context
     * @param statements the parsed statements (mutable)
     */
    protected void performSemanticAnalysis(ParserContext context, TStatementList statements) {
        // Default implementation: no semantic analysis
        // Subclasses can override for vendor-specific behavior
    }

    /**
     * Perform interpretation/evaluation on parsed statements.
     *
     * <p><b>Hook Method:</b> Default implementation does nothing.
     * Override to provide AST interpretation/evaluation.
     *
     * <p><b>Typical Implementation:</b>
     * <ul>
     *   <li>Execute simple SQL statements</li>
     *   <li>Evaluate expressions</li>
     *   <li>Constant folding</li>
     *   <li>Static analysis</li>
     * </ul>
     *
     * @param context the parser context
     * @param statements the parsed statements (mutable)
     */
    protected void performInterpreter(ParserContext context, TStatementList statements) {
        // Default implementation: no interpreter
        // Subclasses can override to provide AST interpretation
    }

    /**
     * Copy error messages from a statement to the parser's error collection.
     *
     * <p>This method should be called by performParsing implementations
     * when a statement has syntax errors.
     *
     * @param statement the statement with errors
     */
    protected void copyErrorsFromStatement(TCustomSqlStatement statement) {
        if (statement == null || statement.getSyntaxErrors() == null) {
            return;
        }

        for (int i = 0; i < statement.getSyntaxErrors().size(); i++) {
            this.syntaxErrors.add(new TSyntaxError((TSyntaxError) statement.getSyntaxErrors().get(i)));
        }
    }

    /**
     * Attempt error recovery for CREATE TABLE/INDEX statements with unsupported options.
     *
     * <p>When parsing CREATE TABLE or CREATE INDEX statements, the parser may encounter
     * vendor-specific options that are not in the grammar. This method implements the
     * legacy error recovery behavior by marking unsupported tokens after the main
     * definition as SQL*Plus commands (effectively ignoring them).
     *
     * <p><b>Recovery Strategy:</b>
     * <ol>
     *   <li>Find the closing ')' of the column/index definitions (nested=0)</li>
     *   <li>Mark all remaining tokens (except ';') as sqlpluscmd to ignore them</li>
     *   <li>Clear errors and re-parse the statement</li>
     * </ol>
     *
     * <p><b>When to call:</b> After parsing a statement that has errors.
     * Only recovers if ENABLE_ERROR_RECOVER_IN_CREATE_TABLE is true.
     *
     * @param statement the statement to attempt recovery on
     * @param parseResult the result code from parsing (0 = success)
     * @param onlyNeedRawParseTree whether only raw parse tree is needed
     * @return new parse result after recovery attempt, or original if no recovery
     */
    protected int attemptErrorRecovery(TCustomSqlStatement statement, int parseResult, boolean onlyNeedRawParseTree) {
        boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;

        if (doRecover && ((parseResult != 0) || (statement.getErrorCount() > 0))) {
            if (((statement.sqlstatementtype == ESqlStatementType.sstcreatetable)
                    || ((statement.sqlstatementtype == ESqlStatementType.sstcreateindex) && (this.vendor != EDbVendor.dbvcouchbase))
                    ) && (!TBaseType.c_createTableStrictParsing)
            ) {
                // Only parse main body of create table/index, ignore unsupported options after closing ')'
                int nested = 0;
                boolean isIgnore = false;
                boolean isFoundIgnoreToken = false;
                TSourceToken firstIgnoreToken = null;

                for (int k = 0; k < statement.sourcetokenlist.size(); k++) {
                    TSourceToken st = statement.sourcetokenlist.get(k);
                    if (isIgnore) {
                        if (st.issolidtoken() && (st.tokencode != ';')) {
                            isFoundIgnoreToken = true;
                            if (firstIgnoreToken == null) {
                                firstIgnoreToken = st;
                            }
                        }
                        if (st.tokencode != ';') {
                            st.tokencode = TBaseType.sqlpluscmd;
                        }
                        continue;
                    }
                    if (st.tokencode == (int) ')') {
                        nested--;
                        if (nested == 0) {
                            // Check if next token is "AS ( SELECT" (table created from select)
                            boolean isSelect = false;
                            TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
                            if (st1 != null) {
                                TSourceToken st2 = st.searchToken((int) '(', 2);
                                if (st2 != null) {
                                    TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
                                    isSelect = (st3 != null);
                                }
                            }
                            if (!isSelect) isIgnore = true;
                        }
                    }
                    if ((st.tokencode == (int) '(') || (st.tokencode == TBaseType.left_parenthesis_2)) {
                        nested++;
                    }
                }

                // For Oracle, validate that ignored tokens are valid table properties
                if ((this.vendor == EDbVendor.dbvoracle) && (firstIgnoreToken != null)
                        && (!TBaseType.searchOracleTablePros(firstIgnoreToken.toString()))) {
                    // Not a valid Oracle table property, don't ignore
                    isFoundIgnoreToken = false;
                }

                if (isFoundIgnoreToken) {
                    statement.clearError();
                    parseResult = statement.parsestatement(null, false, onlyNeedRawParseTree);
                }
            }
        }

        return parseResult;
    }

    /**
     * Get the syntax errors collected during parsing.
     *
     * @return list of syntax errors (never null)
     */
    public List<TSyntaxError> getSyntaxErrors() {
        return syntaxErrors;
    }

    /**
     * Get the count of syntax errors.
     *
     * @return number of syntax errors
     */
    public int getErrorCount() {
        return syntaxErrors.size();
    }

    /**
     * Check if a token is a dollar function delimiter ($$, $tag$, etc.) for PostgreSQL-family databases.
     * <p>
     * Migrated from TGSqlParser.isDollarFunctionDelimiter() (lines 5074-5080).
     * <p>
     * Dollar-quoted strings are used in PostgreSQL-family databases to delimit function bodies.
     * Each vendor has its own delimiter token code.
     *
     * @param tokencode the token code to check
     * @param dbVendor the database vendor
     * @return true if the token is a dollar function delimiter for the given vendor
     */
    protected boolean isDollarFunctionDelimiter(int tokencode, EDbVendor dbVendor) {
        return ((tokencode == TBaseType.rrw_postgresql_function_delimiter) && (dbVendor == EDbVendor.dbvpostgresql))
                || ((tokencode == TBaseType.rrw_postgresql_function_delimiter) && (dbVendor == EDbVendor.dbvduckdb))
                || ((tokencode == TBaseType.rrw_greenplum_function_delimiter) && (dbVendor == EDbVendor.dbvgreenplum))
                || ((tokencode == TBaseType.rrw_redshift_function_delimiter) && (dbVendor == EDbVendor.dbvredshift))
                || ((tokencode == TBaseType.rrw_snowflake_function_delimiter) && (dbVendor == EDbVendor.dbvsnowflake));
    }

    /**
     * Mantis 4497: classify a '/' that was tentatively flagged as an Oracle
     * SQL*Plus statement terminator (tokencode {@link TBaseType#sqlpluscmd}, but
     * tokentype still {@link ETokenType#ttslash}) by the vendor tokenizer.
     * <p>
     * PostgreSQL/psql, Redshift and Greenplum have no '/' statement terminator --
     * '/' is purely the division operator. A genuine SQL*Plus terminator alone on
     * a line is only ever followed by end-of-input or the start of a brand-new
     * statement; it is never followed by an operand. Conversely, a division slash
     * is always followed by its right-hand operand, which can be an identifier, a
     * literal, '(', a unary sign, or even a keyword that begins an expression
     * (NULL, CAST, CASE, ...).
     * <p>
     * We therefore treat the slash as division for everything except the two
     * unambiguous terminator signals: end-of-input, or a following keyword from a
     * conservative allowlist of strongly-reserved DML/DDL/TCL statement verbs
     * (see {@link #isStatementStartKeyword(int)}). This keeps the common
     * Oracle-style multi-statement script ({@code CREATE ... / CREATE ... /})
     * splitting at the slash while letting genuine division expressions parse as a
     * single statement.
     * <p>
     * <b>Known limitations of this next-token heuristic</b> (both are acceptable
     * because these dialects do not natively support a '/' terminator at all, so
     * the terminator handling is only a courtesy for pasted Oracle scripts):
     * <ul>
     *   <li>A bare '/' followed by a <i>non-reserved</i> command word that is not
     *       on the allowlist (e.g. {@code COPY}, {@code VACUUM}, {@code VALUES})
     *       is treated as division rather than a terminator, so such scripts are
     *       not split there. Those words are deliberately excluded because they
     *       are also valid unquoted identifiers and would otherwise break real
     *       division expressions whose right operand is such a column.</li>
     *   <li>Conversely, dividing by a column whose unquoted name happens to be one
     *       of the strongly-reserved verbs on the allowlist (e.g. a column literally
     *       named {@code select} or {@code create}) would be misread as a
     *       terminator. Such column names are effectively never used in practice.</li>
     * </ul>
     *
     * @param slash the slash token (tokentype ttslash) to classify
     * @return true if the slash is a division operator, false if it terminates
     */
    protected boolean isDivisionOperatorContext(TSourceToken slash) {
        TSourceToken next = slash.nextSolidToken();
        if (next == null) return false; // end of input -> genuine terminator
        return !isStatementStartKeyword(next.tokencode);
    }

    /**
     * Returns true when the token code is a strongly-reserved DML/DDL/TCL verb
     * that, in practice, only ever begins a new SQL statement and is never used as
     * a bare unquoted column identifier. Used by
     * {@link #isDivisionOperatorContext(TSourceToken)} to tell a genuine SQL*Plus
     * '/' terminator from the division operator.
     * <p>
     * Non-reserved command words that double as plausible identifiers (SET, SHOW,
     * RESET, ANALYZE, LOCK, COPY, VACUUM, VALUES, COMMENT, ...) are intentionally
     * excluded: treating them as terminators would break valid division
     * expressions whose right operand is such a column.
     */
    private boolean isStatementStartKeyword(int tokencode) {
        return tokencode == TBaseType.rrw_select
                || tokencode == TBaseType.rrw_insert
                || tokencode == TBaseType.rrw_update
                || tokencode == TBaseType.rrw_delete
                || tokencode == TBaseType.rrw_merge
                || tokencode == TBaseType.rrw_create
                || tokencode == TBaseType.rrw_alter
                || tokencode == TBaseType.rrw_drop
                || tokencode == TBaseType.rrw_truncate
                || tokencode == TBaseType.rrw_grant
                || tokencode == TBaseType.rrw_revoke
                || tokencode == TBaseType.rrw_explain
                || tokencode == TBaseType.rrw_with
                || tokencode == TBaseType.rrw_begin
                || tokencode == TBaseType.rrw_declare
                || tokencode == TBaseType.rrw_call
                || tokencode == TBaseType.rrw_do
                || tokencode == TBaseType.rrw_commit
                || tokencode == TBaseType.rrw_rollback
                || tokencode == TBaseType.rrw_savepoint;
    }

    /**
     * Hook method called when a raw statement is complete.
     * <p>
     * This method is called by vendor-specific raw statement extraction methods
     * (e.g., dooraclegetrawsqlstatements) when a statement boundary is detected.
     * It sets up the statement with parser references and adds it to the statement list.
     *
     * @param context parser context
     * @param statement the completed statement
     * @param mainParser main parser instance
     * @param secondaryParser secondary parser instance (may be null)
     * @param statementList statement list to add to
     * @param isLastStatement true if this is the last statement
     * @param builder optional result builder (used during raw statement extraction, may be null)
     */
    protected void onRawStatementComplete(ParserContext context,
                                          TCustomSqlStatement statement,
                                          TCustomParser mainParser,
                                          TCustomParser secondaryParser,
                                          TStatementList statementList,
                                          boolean isLastStatement,
                                          SqlParseResult.Builder builder) {
        if (statement == null || statementList == null) {
            return;
        }

        // CRITICAL: Set gsqlparser reference NOW (before parsing) so nested statements
        // can access parser's dbvendor via getGsqlparser().getDbVendor()
        // This matches legacy behavior from doongetrawsqlstatementevent()
        if (context != null && context.getGsqlparser() != null) {
            // Cast to TGSqlParser - we know the type from buildContext()
            statement.setGsqlparser((gudusoft.gsqlparser.TGSqlParser) context.getGsqlparser());
        }
        statement.parser = mainParser;
        statement.plsqlparser = secondaryParser;

        if (statement.sourcetokenlist != null && statement.sourcetokenlist.size() > 0) {
            TSourceToken startToken = statement.sourcetokenlist.get(0);
            TSourceToken endToken = statement.sourcetokenlist.get(statement.sourcetokenlist.size() - 1);

            statement.setStartToken(startToken);
            statement.setEndToken(endToken);

            // Always set lastTokenOfStatementBeenValidated when a statement is found
            // This ensures getLastLineNoOfLastStatementBeenValidated() returns valid line number
            // for any successfully identified statement, regardless of whether it ends with semicolon
            if (context != null && endToken != null && builder != null) {
                builder.lastTokenOfStatementBeenValidated(endToken);
            }
        }

        // Vendor-specific statement completion logic (migrated from TGSqlParser.doongetrawsqlstatementevent lines 5129-5178)
        onRawStatementCompleteVendorSpecific(statement);

        statementList.add(statement);

    }

    /**
     * Hook for vendor-specific logic when a raw statement is completed.
     * <p>
     * Migrated from TGSqlParser.doongetrawsqlstatementevent() (lines 5129-5178).
     * <p>
     * This method is called after basic statement setup but before adding to the statement list.
     * Subclasses can override to add vendor-specific token manipulations or metadata.
     * <p>
     * Default implementation handles PostgreSQL-family routine body processing.
     *
     * @param statement the completed statement
     */
    protected void onRawStatementCompleteVendorSpecific(TCustomSqlStatement statement) {
        // Handle PostgreSQL-family databases: Mark non-SQL/PLSQL routine body tokens
        // Migrated from TGSqlParser.doongetrawsqlstatementevent() lines 5143-5178
        if (((this.vendor == EDbVendor.dbvpostgresql) || (this.vendor == EDbVendor.dbvgreenplum)
                || (this.vendor == EDbVendor.dbvredshift) || (this.vendor == EDbVendor.dbvsnowflake))
                && (statement instanceof TRoutine)) {

            TRoutine routine = (TRoutine) statement;
            if (!routine.isBodyInSQL()) {
                TSourceToken st;
                boolean inBody = false;
                StringBuilder routineBodyBuilder = new StringBuilder();

                for (int i = 0; i < statement.sourcetokenlist.size(); i++) {
                    st = statement.sourcetokenlist.get(i);

                    // Check for dollar function delimiter ($$, $tag$, etc.)
                    if (isDollarFunctionDelimiter(st.tokencode, this.vendor)) {
                        if (!inBody) {
                            inBody = true;
                            routineBodyBuilder.setLength(0);
                            routineBodyBuilder.append(st.toString());
                        } else {
                            inBody = false;
                            routineBodyBuilder.append(st.toString());
                            break;
                        }
                        continue;
                    }

                    if (inBody) {
                        // Mark body tokens as sqlpluscmd so they're not parsed as SQL
                        st.tokencode = TBaseType.sqlpluscmd;
                        routineBodyBuilder.append(st.toString());
                    }
                }

                routine.setRoutineBody(routineBodyBuilder.toString());
            }
        }
    }

    private static final int ENCODING_UTF16 = 1;
    private static final int ENCODING_UTF32 = 2;
    private static final int ENCODING_UTF8_BOM = 3;

    /**
     * Trial size verdict for the input currently being prepared, set by
     * {@link #prepareSqlReader} and consumed by {@link #tokenize}. Always null
     * in full builds.
     */
    private String trialRefusalMessage = null;

    /**
     * Resolve the active input source and hand back a reader over it.
     *
     * <p>This is the one place where bytes become characters, so it is also the
     * one place the trial cap is measured -- in RAW BYTES, before decoding.
     * A byte-backed source is read once into a bounded snapshot; when the
     * snapshot is within the cap it IS the whole input, and the reader is built
     * over it so the source is never read twice. Measuring anywhere downstream
     * would mean re-encoding decoded text, which is what every earlier revision
     * of this gate got wrong.
     */
    protected PreparedSqlReader prepareSqlReader(ParserContext context) throws IOException {
        BufferedReader reader;
        String effectiveCharset = context.getSqlCharset();

        if (context.getSqlText() != null) {
            // A String has no wire format: measured as characters, which keeps
            // any re-parse of already-ingested text (a routine body, dynamic
            // SQL) inside the cap automatically -- see TrialInputGuard.
            TrialInputGuard.Result verdict = TrialInputGuard.checkText(context.getSqlText());
            if (verdict.isRefused()) {
                trialRefusalMessage = verdict.getRefusalMessage();
                return new PreparedSqlReader(new BufferedReader(new StringReader("")), effectiveCharset);
            }
            reader = new BufferedReader(new StringReader(context.getSqlText()));
            return new PreparedSqlReader(reader, effectiveCharset);
        }

        if (context.getSqlFilename() != null && !context.getSqlFilename().isEmpty()) {
            FileInputStream fileStream = new FileInputStream(context.getSqlFilename());
            // Deliberately NOT File.length(): the file can change between a stat
            // and the read, and the bytes that decide the verdict must be the
            // same bytes that feed decoding.
            InputStream measured = applyTrialGuard(fileStream);
            // We opened this stream, so we close it whenever the guard stops
            // using it: on a refusal, and when a snapshot replaces it. A
            // caller-supplied stream (branch below) is never closed here.
            if (measured == null) {
                fileStream.close();
                return new PreparedSqlReader(new BufferedReader(new StringReader("")), effectiveCharset);
            }
            if (measured != fileStream) {
                fileStream.close();
            }
            BufferedInputStream bufferedStream = new BufferedInputStream(measured, 8);
            int encodingType = detectEncodingFromBom(bufferedStream);
            String charsetToUse = resolveCharsetName(encodingType, context.getSqlCharset());
            InputStreamReader streamReader = new InputStreamReader(bufferedStream, charsetToUse);
            reader = new BufferedReader(streamReader);
            skipBomIfPresent(reader, encodingType);
            return new PreparedSqlReader(reader, charsetToUse);
        }

        InputStream contextStream = context.getSqlInputStream();
        if (contextStream != null) {
            contextStream = applyTrialGuard(contextStream);
            if (contextStream == null) {
                return new PreparedSqlReader(new BufferedReader(new StringReader("")), effectiveCharset);
            }
            BufferedInputStream bufferedStream = (contextStream instanceof BufferedInputStream)
                ? (BufferedInputStream) contextStream
                : new BufferedInputStream(contextStream, 8);
            int encodingType = detectEncodingFromBom(bufferedStream);
            String charsetToUse = resolveCharsetName(encodingType, context.getSqlCharset());
            InputStreamReader streamReader = new InputStreamReader(bufferedStream, charsetToUse);
            reader = new BufferedReader(streamReader);
            skipBomIfPresent(reader, encodingType);
            return new PreparedSqlReader(reader, charsetToUse);
        }

        // Default: empty input is valid, return reader for empty string
        reader = new BufferedReader(new StringReader(""));
        return new PreparedSqlReader(reader, effectiveCharset);
    }

    /**
     * Measure a byte source against the trial cap and return the stream the
     * caller should actually read: the captured snapshot on acceptance, or null
     * when the input is refused (with {@link #trialRefusalMessage} set). Returns
     * the source untouched in full builds.
     *
     * <p>The snapshot is operation-local. It is never stored as parser state, so
     * a pooled or reused parser cannot serve a later input from an earlier
     * capture.
     */
    private InputStream applyTrialGuard(InputStream source) throws IOException {
        if (TBaseType.full_edition) {
            return source;
        }
        TrialInputGuard.Result verdict = TrialInputGuard.checkStream(source);
        if (verdict.isIoFailure()) {
            // A partial read that then failed is not a small legal input.
            throw verdict.getIoFailure();
        }
        if (verdict.isRefused()) {
            trialRefusalMessage = verdict.getRefusalMessage();
            return null;
        }
        byte[] snapshot = verdict.getSnapshot();
        return snapshot == null ? source : new ByteArrayInputStream(snapshot);
    }

    private int detectEncodingFromBom(BufferedInputStream stream) throws IOException {
        if (stream == null || !stream.markSupported()) {
            return 0;
        }

        byte[] bom = new byte[4];
        stream.mark(bom.length + 1);
        int read = stream.read(bom, 0, bom.length);
        stream.reset();

        if (read < 2) {
            return 0;
        }

        if (((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE))
            || ((bom[0] == (byte) 0xFE) && (bom[1] == (byte) 0xFF))) {
            if (read >= 4 && (((bom[2] == (byte) 0xFF) && (bom[3] == (byte) 0xFE))
                || ((bom[2] == (byte) 0xFE) && (bom[3] == (byte) 0xFF)))) {
                return ENCODING_UTF32;
            }
            return ENCODING_UTF16;
        }

        if (read >= 3 && (bom[0] == (byte) 0xEF) && (bom[1] == (byte) 0xBB) && (bom[2] == (byte) 0xBF)) {
            return ENCODING_UTF8_BOM;
        }

        return 0;
    }

    private String resolveCharsetName(int encodingType, String contextCharset) {
        switch (encodingType) {
            case ENCODING_UTF16:
                return "UTF-16";
            case ENCODING_UTF32:
                return "UTF-32";
            case ENCODING_UTF8_BOM:
                return "UTF-8";
            default:
                if (contextCharset != null && !contextCharset.isEmpty()) {
                    return contextCharset;
                }
                return Charset.defaultCharset().name();
        }
    }

    private void skipBomIfPresent(BufferedReader reader, int encodingType) throws IOException {
        if (encodingType != ENCODING_UTF8_BOM || reader == null || !reader.markSupported()) {
            return;
        }

        reader.mark(1);
        int ch = reader.read();
        if (ch != 0xFEFF && ch != -1) {
            reader.reset();
        }
    }

    // ========== Utility Methods ==========

    /**
     * Initialize global context and frame stack for statement parsing.
     * <p>
     * This method sets up the semantic analysis infrastructure required during
     * the parsing phase. It creates:
     * <ul>
     *   <li>Global context (TContext) for semantic analysis</li>
     *   <li>SQL environment (TSQLEnv) with vendor-specific configuration</li>
     *   <li>Frame stack for scope management</li>
     *   <li>Global scope frame as the outermost scope</li>
     * </ul>
     *
     * <p><b>When to call:</b> At the beginning of performParsing(), before parsing statements.
     *
     * <p><b>Cleanup required:</b> Must call {@code globalFrame.popMeFromStack(frameStack)}
     * after all statements are parsed to clean up the frame stack.
     *
     * <p><b>Extracted from:</b> Identical implementations in OracleSqlParser and MssqlSqlParser
     * to eliminate ~16 lines of duplicate code per parser.
     */
    protected void initializeGlobalContext() {
        // Initialize global context for semantic analysis
        this.globalContext = new TContext();
        this.sqlEnv = new TSQLEnv(this.vendor) {
            @Override
            public void initSQLEnv() {
                // Vendor-specific initialization can be added by subclasses if needed
            }
        };
        this.globalContext.setSqlEnv(this.sqlEnv, this.sqlstatements);

        // Create global scope frame
        this.frameStack = new java.util.Stack<TFrame>();
        TGlobalScope globalScope = new TGlobalScope();
        globalScope.resetCurrentStmtIndex();
        globalScope.setSqlEnv(this.sqlEnv);
        this.globalFrame = new TFrame(globalScope);
        this.globalFrame.pushMeToStack(this.frameStack);
    }

    /**
     * Handle exceptions that occur during individual statement parsing.
     * <p>
     * This method provides robust error handling that allows parsing to continue
     * even when individual statements throw exceptions. It:
     * <ul>
     *   <li>Creates a detailed {@link TSyntaxError} with exception information</li>
     *   <li>Captures statement location (line, column) from first token</li>
     *   <li>Includes statement number, exception type, and message</li>
     *   <li>Optionally logs full stack trace if debugging is enabled</li>
     *   <li>Adds error to {@link #syntaxErrors} list for user feedback</li>
     * </ul>
     *
     * <p><b>Benefits:</b>
     * <ul>
     *   <li>Parsing continues for remaining statements after exception</li>
     *   <li>Users get complete error feedback for all statements</li>
     *   <li>Developers get stack traces for debugging parser issues</li>
     * </ul>
     *
     * <p><b>Example error message:</b><br>
     * {@code "Exception during parsing statement 3: NullPointerException - Cannot invoke..."}
     *
     * <p><b>Extracted from:</b> Identical implementations in OracleSqlParser and MssqlSqlParser
     * to eliminate ~51 lines of duplicate code per parser.
     *
     * @param stmt the statement that failed to parse
     * @param statementIndex 0-based index of the statement in the statement list
     * @param ex the exception that was thrown during parsing
     */
    protected void handleStatementParsingException(TCustomSqlStatement stmt, int statementIndex, Exception ex) {
        // Create user-friendly error message with context
        String errorMsg = String.format("Exception during parsing statement %d: %s - %s",
            statementIndex + 1,  // Convert to 1-based for user readability
            ex.getClass().getSimpleName(),
            ex.getMessage() != null ? ex.getMessage() : "No details");

        // Get first token of statement for error location
        TSourceToken firstToken = null;
        if (stmt.sourcetokenlist != null && stmt.sourcetokenlist.size() > 0) {
            firstToken = stmt.sourcetokenlist.get(0);
        }

        // Create syntax error with exception details
        TSyntaxError syntaxError;
        if (firstToken != null) {
            // Use token location for accurate error reporting
            syntaxError = new TSyntaxError(
                firstToken.getAstext(),
                firstToken.lineNo,
                firstToken.columnNo,
                errorMsg,
                EErrorType.sperror,
                TBaseType.MSG_ERROR_SYNTAX_ERROR,
                stmt,
                firstToken.posinlist
            );
        } else {
            // Fallback if no token info available
            syntaxError = new TSyntaxError(
                "",
                0,
                0,
                errorMsg,
                EErrorType.sperror,
                TBaseType.MSG_ERROR_SYNTAX_ERROR,
                stmt,
                -1
            );
        }

        this.syntaxErrors.add(syntaxError);

        // Log to console if debugging enabled
        if (TBaseType.DUMP_RESOLVER_LOG_TO_CONSOLE) {
            System.err.println("ERROR: " + errorMsg);
            ex.printStackTrace();
        }
    }

    /**
     * Hook method for vendor-specific post-processing after a statement is parsed.
     * <p>
     * This method is called after each statement is successfully parsed but before
     * error recovery and error collection. Subclasses can override this to perform
     * vendor-specific operations such as:
     * <ul>
     *   <li>Checking for vendor-specific syntax errors in nested statements</li>
     *   <li>Validating vendor-specific constraints</li>
     *   <li>Collecting vendor-specific metadata</li>
     * </ul>
     *
     * <p><b>Default implementation:</b> Does nothing (no-op).
     *
     * <p><b>Example override (Oracle):</b><br>
     * <pre>{@code
     * @Override
     * protected void afterStatementParsed(TCustomSqlStatement stmt) {
     *     if (stmt.isoracleplsql()) {
     *         findAllSyntaxErrorsInPlsql(stmt);
     *     }
     * }
     * }</pre>
     *
     * <p><b>When called:</b> After {@code stmt.parsestatement()} succeeds,
     * before {@code handleCreateTableErrorRecovery()} and {@code copyErrorsFromStatement()}.
     *
     * @param stmt the statement that was just parsed
     */
    protected void afterStatementParsed(TCustomSqlStatement stmt) {
        // Default: no additional processing
        // Subclasses override to add vendor-specific post-processing
    }

    /**
     * Get next source token from the lexer.
     * <p>
     * This method wraps the lexer's yylexwrap() call and performs several important tasks:
     * <ul>
     *   <li>Fetches the next raw token from the lexer</li>
     *   <li>Combines consecutive whitespace/newline tokens for cleaner token stream</li>
     *   <li>Sets token metadata (vendor, status, container, position in list)</li>
     *   <li>Optionally calls token handler callback</li>
     * </ul>
     *
     * <p><b>Token Consolidation Rules:</b>
     * <ul>
     *   <li>Whitespace after a newline is merged into the newline token</li>
     *   <li>Consecutive newlines are merged into a single newline token</li>
     * </ul>
     *
     * <p><b>Implementation Note:</b>
     * This method is extracted from TGSqlParser.getanewsourcetoken() and made
     * available to all database-specific parsers to avoid code duplication.
     *
     * @return next source token, or null if end of input
     */
    protected TSourceToken getanewsourcetoken() {
        TSourceToken pst = null, prevst;

        while (true) {
            pst = new TSourceToken("");
            if (lexer.yylexwrap(pst) == 0) {
                pst = null;
                break;
            }

            pst.setDbvendor(vendor);
            pst.tokenstatus = ETokenStatus.tsoriginal;

            if (pst.tokentype == ETokenType.ttreturn) {
                pst.setAstext(towinlinebreak(pst.getAstext()));
            }

            // Combine space & linebreak after a linebreak into one
            if ((pst.tokentype == ETokenType.ttwhitespace)
                    && (sourcetokenlist.curpos >= 0)) {
                prevst = sourcetokenlist.get(sourcetokenlist.curpos);
                if (prevst.tokentype == ETokenType.ttreturn) {
                    // Can't discard whitespace after linebreak, it will be used
                    // to judge whether / at the beginning of the line is a sqlplus cmd or not
                    // check isValidPlaceForDivToSqlplusCmd for more
                    prevst.setAstext(prevst.getAstext() + pst.getAstext());
                    continue;
                }
            }

            // Combine consecutive newlines
            if ((pst.tokentype == ETokenType.ttreturn)
                    && (sourcetokenlist.curpos >= 0)) {
                prevst = sourcetokenlist.get(sourcetokenlist.curpos);

                if (prevst.tokentype == ETokenType.ttreturn) {
                    prevst.setAstext(prevst.getAstext() + pst.getAstext());
                    continue;
                }

                // Note: The original code has a commented section about merging
                // whitespace with newline. We're preserving the behavior here
                // which does NOT merge preceding whitespace with newline.
            }

            break;
        }

        if (pst != null) {
            pst.container = sourcetokenlist;
            sourcetokenlist.curpos = sourcetokenlist.curpos + 1;
            pst.posinlist = sourcetokenlist.curpos;

            // Optional token handler callback. Fires inline, unchanged: an
            // over-cap input is refused before tokenization starts, so no token
            // can reach the callback from a rejected input and there is nothing
            // to defer.
            if (tokenHandle != null) {
                tokenHandle.processToken(pst);
            }
        }

        lexer.setTokenTableValue(pst);
        return pst;
    }

    /**
     * Convert line breaks to Windows format.
     * <p>
     * Currently returns the input unchanged. This method exists for compatibility
     * with the original TGSqlParser implementation.
     *
     * @param s Input string
     * @return String with Windows line breaks (currently unchanged)
     */
    protected String towinlinebreak(String s) {
        return s;
        // if (s == null) return null;
        // return s.replace("\n", "\r\n");
    }

    /**
     * Get the delimiter character for this vendor.
     *
     * @return delimiter character (e.g., ';', '/', '$')
     */
    public char getDelimiterChar() {
        return delimiterChar;
    }

    /**
     * Get the default delimiter string for this vendor.
     *
     * @return default delimiter string
     */
    public String getDefaultDelimiterStr() {
        return defaultDelimiterStr;
    }

    @Override
    public String toString() {
        return getClass().getSimpleName() + "{vendor=" + vendor + "}";
    }
}