package gudusoft.gsqlparser;

import gudusoft.gsqlparser.compiler.TASTEvaluator;
import gudusoft.gsqlparser.compiler.TContext;
import gudusoft.gsqlparser.compiler.TGlobalScope;
import gudusoft.gsqlparser.compiler.TFrame;
import gudusoft.gsqlparser.nodes.*;
import gudusoft.gsqlparser.nodes.teradata.TTeradataHelper;
import gudusoft.gsqlparser.resolver.*;
import gudusoft.gsqlparser.resolver2.TSQLResolver2;
import gudusoft.gsqlparser.resolver2.TSQLResolverConfig;
import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
import gudusoft.gsqlparser.sqlenv.TSQLEnv;
import gudusoft.gsqlparser.stmt.*;
import gudusoft.gsqlparser.stmt.dax.TDaxEvaluateStmt;
import gudusoft.gsqlparser.stmt.dax.TDaxExprStmt;
import gudusoft.gsqlparser.stmt.greenplum.TSlashCommand;
import gudusoft.gsqlparser.stmt.mssql.TMssqlBlock;
import gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure;
import gudusoft.gsqlparser.stmt.mssql.TMssqlExecute;
import gudusoft.gsqlparser.stmt.mysql.TMySQLSource;
import gudusoft.gsqlparser.stmt.oracle.TPlsqlCreatePackage;
import gudusoft.gsqlparser.stmt.oracle.TSqlplusCmdStatement;
import gudusoft.gsqlparser.stmt.snowflake.TCreateTaskStmt;
import gudusoft.gsqlparser.stmt.teradata.TTeradataBTEQCmd;
import gudusoft.gsqlparser.stmt.teradata.TTeradataFastExportCmd;
import gudusoft.gsqlparser.stmt.teradata.TTeradataFastLoadCmd;
import gudusoft.gsqlparser.stmt.teradata.TTeradataMultiLoadCmd;
import gudusoft.gsqlparser.stmt.teradata.utilities.BteqCmdType;
import gudusoft.gsqlparser.stmt.teradata.utilities.TeradataUtilityType;
import gudusoft.gsqlparser.util.TSnowflakeParameterChecker;
import gudusoft.gsqlparser.parser.SqlParser;
import gudusoft.gsqlparser.parser.ParserContext;
import gudusoft.gsqlparser.parser.SqlParseResult;
import gudusoft.gsqlparser.parser.AbstractSqlParser;


import java.io.*;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.*;

import static gudusoft.gsqlparser.ESqlStatementType.*;


/**
 * This is the first class people start to use this SQL parser library.
 * This class includes a lexer and a parser. The lexer is used to tokenize the input SQL, turn the input SQL text
 * into a list of source tokens. The parser use this list source token as input, using the grammar rule of the specified
 * database vendor to check the syntax of the input SQL and build the parse tree of this SQL if there is no syntax error
 * in the input SQL.
 * <p></p>
 * Creating a SQL parser by specifing {@link gudusoft.gsqlparser.EDbObjectType a database vendor},
 * then set SQL script text via {@link #setSqltext} method or reading the input SQL from a file via
 * {@link #setSqlfilename} method.
 * <p></p>
 * After that, call one of the following methods to achieve what you need:
 * <ul>
 *  <li>{@link #tokenizeSqltext}, turns the input SQL into a sequence of token which is the
 *  basic lexis element of SQL syntax. Token is categorized as keyword, identifier,
 *  number, operator, whitespace and other types. All source tokens can be fetched
 *  via the {@link #getSourcetokenlist()} method</li>
 *
 *  <li>{@link #getrawsqlstatements}, separates the SQL statements in the input SQL script without
 *  doing syntax check, use the {@link #getSqlstatements()} method to get a list of SQL statements
 *  which is the sub-class of {@link TCustomSqlStatement}, get SQL statement type
 *  via the {@link TCustomSqlStatement#sqlstatementtype} field, and string representation of
 *  each SQL statement via the {@link TCustomSqlStatement#toString} method. All source tokens in this SQL statement
 *  is available by using {@link TCustomSqlStatement#sourcetokenlist} filed.
 *  Since no parse tree is built by calling this method, no further detailed information about the SQL statement is available.
 *  </li>
 *
 *  <li>{@link #parse}, Check syntax of the input SQL, doing some kind of semantic analysis without connecting to
 *  a real database.
 *  This method will do a in-depth analysis of the input SQL such as building the link between table and columns.
 *  The parse tree of the input SQL is available after calling this method.
 *  </li>
 *  </ul>
 *
 *  The parser checks the syntax of those SQL statements one by one. If syntax error is found in a SQL statement,
 *  an error will be logged, no parse tree will be built for this SQL statement,
 *  the error message can be fetched using the {@link #getErrormessage()} method.
 *  <p></p>
 *  The syntax error in one SQL statement doesn't prevent the parser continue to check the syntax of the next SQL statement.
 *  After checking syntax of all SQL statements, use the {@link #getErrorCount()} method to get the total number of errors.
 *  <p></p>
 *  A syntax error in a SQL stored procedure will cease this parser to check syntax of the rest SQL statements
 *  in this stored procedure.
 *
 * <p>Format SQL script can be done after calling {@link #parse()}.
 * <code>
 *
 *      int ret = sqlparser.parse();
 *       if (ret == 0){
 *           GFmtOpt option = GFmtOptFactory.newInstance();
 *           String result = FormatterFactory.pp(sqlparser, option);
 *           System.out.println(result);
 *       }else{
 *           System.out.println(sqlparser.getErrormessage());
 *       }
 *
 * </code>
 *
 * <p> After paring SQL script, all parse tree nodes are available for use, some of use cases
 * are:
 * <ul>
 *     <li>Table/column impact analysis</li>
 *     <li>SQL rewriting</li>
 *     <li>SQL translate between different databases</li>
 *     <li>SQL migration analysis</li>
 *     <li>Help to anti SQL injection</li>
 *     <li><a href="http://support.sqlparser.com/">More use cases</a></li>
 *     </ul>
 *
 * <p>Typically, SQL parse tree nodes generated by this SQL Parser were closely related to SQL
 * elements defined in database vendor's SQL reference book. here is a brief summary of some
 * most used SQL elements and corresponding classes defined in this SQL parser.
 * <ul>
 *     <li>SQL identifier: {@link gudusoft.gsqlparser.nodes.TObjectName}</li>
 *     <li>SQL literal: {@link gudusoft.gsqlparser.nodes.TConstant}</li>
 *     <li>SQL datatype: {@link gudusoft.gsqlparser.nodes.TTypeName}</li>
 *     <li>SQL function: {@link gudusoft.gsqlparser.nodes.TFunctionCall}</li>
 *     <li>SQL constraint: {@link gudusoft.gsqlparser.nodes.TConstraint}</li>
 *     <li>SQL expression/condition: {@link gudusoft.gsqlparser.nodes.TExpression}</li>
 *     <li>SQL select list item: {@link gudusoft.gsqlparser.nodes.TResultColumn}</li>
 *      <li>More: {@link gudusoft.gsqlparser.nodes}</li>
 * </ul>
 *
 * <p> Some major SQL statements:
 * <ul>
 *      <li>Select: {@link gudusoft.gsqlparser.stmt.TSelectSqlStatement}</li>
 *      <li>Delete: {@link gudusoft.gsqlparser.stmt.TDeleteSqlStatement}</li>
 *      <li>Insert: {@link gudusoft.gsqlparser.stmt.TInsertSqlStatement}</li>
 *      <li>Update: {@link gudusoft.gsqlparser.stmt.TUpdateSqlStatement}</li>
 *      <li>Create table: {@link gudusoft.gsqlparser.stmt.TCreateTableSqlStatement}</li>
 *      <li>More: {@link   gudusoft.gsqlparser.stmt}</li>
 * </ul>
 *
 * <p>Stored procedure</p>
 * <ul>
 *     <li>Create function: {@link gudusoft.gsqlparser.stmt.db2.TDb2CreateFunction },
 *          {@link gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.mysql.TMySQLCreateFunction},
 *          {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateFunction}</li>
 *     <li>Create procedure: {@link gudusoft.gsqlparser.stmt.db2.TDb2CreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.mysql.TMySQLCreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateProcedure}</li>
 *     <li>Create trigger: {@link gudusoft.gsqlparser.stmt.TCreateTriggerStmt},
 *          {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateTrigger}</li>
 *     <li>Create package: {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreatePackage}</li>
 * </ul>
 *
 * For all available SQL parse tree node classes, please check the API reference.
 *
 *
 */
public class TGSqlParser {




    public void setSqlCharset(String sqlCharset) {
        this.sqlCharset = sqlCharset;
    }

    public String getSqlCharset() {
        return sqlCharset;
    }

    private String sqlCharset = null;

    public static EDbVendor currentDBVendor = EDbVendor.dbvoracle;


    /**
     * Turn the string name of database to dbvendor
     * <ul>
     *     <li>access: EDbVendor.dbvaccess</li>
     *     <li>ansi: EDbVendor.dbvansi</li>
     *     <li>bigquery: EDbVendor.dbvbigquery</li>
     *     <li>clickhouse: EDbVendor.dbvclickhouse</li>
     *     <li>couchbase: EDbVendor.dbvcouchbase</li>
     *     <li>dax: EDbVendor.dbvdax</li>
     *     <li>db2: EDbVendor.dbvdb2</li>
     *     <li>firebird: EDbVendor.dbvfirebird</li>
     *     <li>generic: EDbVendor.dbvgeneric</li>
     *     <li>greenplum: EDbVendor.dbvgreenplum</li>
     *     <li>hana: EDbVendor.dbvhana</li>
     *     <li>hive: EDbVendor.dbvhive</li>
     *     <li>impala: EDbVendor.dbvimpala</li>
     *     <li>informix: EDbVendor.dbvinformix</li>
     *     <li>mdx: EDbVendor.dbvmdx</li>
     *     <li>mssql or sqlserver: EDbVendor.dbvmssql</li>
     *     <li>mysql: EDbVendor.dbvmysql</li>
     *     <li>netezza: EDbVendor.dbvnetezza</li>
     *     <li>odbc: EDbVendor.dbvodbc</li>
     *     <li>openedge: EDbVendor.dbvopenedge</li>
     *     <li>oracle: EDbVendor.dbvoracle</li>
     *     <li>postgresql or postgres: EDbVendor.dbvpostgresql</li>
     *     <li>redshift: EDbVendor.dbvredshift</li>
     *     <li>snowflake: EDbVendor.dbvsnowflake</li>
     *     <li>sybase: EDbVendor.dbvsybase</li>
     *     <li>teradata: EDbVendor.dbvteradata</li>
     *     <li>vertica: EDbVendor.dbvvertica</li>
     * </ul>
     * @param dbVendorName
     * @return dbvendor
     */
    public static EDbVendor getDBVendorByName(String dbVendorName){
        return  EDbVendor.valueOfWithDefault(dbVendorName);
    }

//    public void  teradataCmds(){
//       // int cnt = 0;
//        //((TLexerTeradata)getFlexer()).
////         for(int i=0;i<TLexerTeradata.bteqCmdList.size();i++){
////             for(int j=0;j<TLexerTeradata.multiLoadCmdList.size();j++){
////                 if (TLexerTeradata.bteqCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.multiLoadCmdList.get(j).toString())){
////                   System.out.println("multiLoad: "+TLexerTeradata.bteqCmdList.get(i).toString());
////                 }
////             }
////             for(int j=0;j<TLexerTeradata.fastExportCmdList.size();j++){
////                 if (TLexerTeradata.bteqCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.fastExportCmdList.get(j).toString())){
////                     System.out.println("fastExport: "+TLexerTeradata.bteqCmdList.get(i).toString());
////                 }
////             }
////             for(int j=0;j<TLexerTeradata.fastLoadCmdList.size();j++){
////                 if (TLexerTeradata.bteqCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.fastLoadCmdList.get(j).toString())){
////                     System.out.println("FastLoad: "+TLexerTeradata.bteqCmdList.get(i).toString());
////                 }
////             }
////         } //bteqCmdList
//
////        for(int i=0;i<TLexerTeradata.fastLoadCmdList.size();i++){
////            for(int j=0;j<TLexerTeradata.multiLoadCmdList.size();j++){
////                if (TLexerTeradata.fastLoadCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.multiLoadCmdList.get(j).toString())){
////                    System.out.println("multiLoad: "+TLexerTeradata.fastLoadCmdList.get(i).toString());
////                }
////            }
////            for(int j=0;j<TLexerTeradata.fastExportCmdList.size();j++){
////                if (TLexerTeradata.fastLoadCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.fastExportCmdList.get(j).toString())){
////                    System.out.println("fastExport: "+TLexerTeradata.fastLoadCmdList.get(i).toString());
////                }
////            }
////
////        }
//
//    }

    private Stack<TFrame> frameStack = null;

    public Stack<TFrame> getFrameStack(){
        if (frameStack == null){
            frameStack = new Stack<TFrame>();
        }

        return  frameStack;
    }

    public void setFrameStack(Stack<TFrame> frameStack) {
        this.frameStack = frameStack;
    }

    void closeFileStream(){
        if (streamFromSqlFile != null) {
            try {
                streamFromSqlFile.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    FileInputStream streamFromSqlFile = null;
    InputStreamReader sqlStreamReader = null;
    /**
     * A sequence of source tokens created by the lexer after tokenize the input SQL
     *
     * @return a sequence of source tokens
     */
    public TSourceTokenList getSourcetokenlist() {
        return sourcetokenlist;
    }

    /**
     * A list of SQL statements created by the parser.
     * If this list is created after calling the {@link #getrawsqlstatements} method, the syntax of each SQL statement
     * is not checked and the parse tree of each statement is not created. If the {@link #parse} method is called to build
     * this SQL statement list, then every thing is ready.
     *
     * @return a list of SQL statement
     */
    public TStatementList getSqlstatements() {
        return sqlstatements;
    }

    enum stored_procedure_status {start,is_as,body,bodyend,end, cursor_declare};
    enum stored_procedure_type {function,procedure,package_spec,package_body, block_with_begin,block_with_declare,
        create_trigger,create_library,cursor_in_package_spec,others};

    static final int stored_procedure_nested_level = 1024;

    /**
    ** The input SQL Text.
    ** If {@link #sqlfilename} is specified, then this field will be ignored.
    */
    public String sqltext;

    /**
     * set the input SQL text, If {@link #sqlfilename} is specified before this method, the parser will using
     * the SQL text in this field instead of read SQL from {@link #sqlfilename}.
     *
     * @param sqltext the input SQL text
     */
    public void setSqltext(String sqltext) {
        this.sqltext = sqltext;
        this.sqlfilename = "";
        this.sqlInputStream = null;
    }

    /**
     *  The SQL text that being processed.
     *
     * @return the SQL text that being processed
     */
    public String getSqltext() {
        return sqltext;
    }


    /**
    ** The input SQL will be read from this file
     *
    ** If field is specified, then {@link #sqltext} will be ignored.
     * This must be the full path to the file, relative path doesn't work.
    */
    public String sqlfilename;


    /**
     * set the filename from which the input SQL will be read.
     *
     * @param sqlfilename the SQL file name from which the input SQL will be read
     */
    public void setSqlfilename(String sqlfilename) {
        this.sqlfilename = sqlfilename;
        this.sqltext = "";
        this.sqlInputStream = null;
    }

    /**
     * The input SQL filename. This parser can process the unicode encoded SQL file.
     *
     * @return the input SQL filename
     */
    public String getSqlfilename() {
        return sqlfilename;
    }

    /**
     * set the InputStream from which SQL will be read.
     * If this method is called, {@link #sqlfilename} and {@link #sqltext} will be ignored.
     *
     * @param sqlInputStream the InputStream from which SQL will be read
     */
    public void setSqlInputStream(InputStream sqlInputStream) {
        if (sqlInputStream instanceof BufferedInputStream){
            this.sqlInputStream = (BufferedInputStream)sqlInputStream;
        }else{
            this.sqlInputStream = new BufferedInputStream(sqlInputStream);
        }

        this.sqlfilename = "";
        this.sqltext = "";
    }

    private BufferedInputStream  sqlInputStream;

    /**
     *  the InputStream from which SQL will be read
     * @return the InputStream from which SQL will be read
     */
    public InputStream getSqlInputStream() {
        return sqlInputStream;
    }

    /**
    ** Tokens generated by lexer from the input SQL script.
     * Tokens are always available even if there are syntax errors in input the SQL script.
    */
    public TSourceTokenList sourcetokenlist;

    /**
    ** SQL statements generated by this parser from the input SQL script.
     * statements are always available even if there are syntax errors in input SQL script.
     * if there is no syntax error in a statement, you can access the parse tree of the statement to fetch more information
     * such as tables, columns, etc.
    */
    public TStatementList sqlstatements;

    /**
     * The TSQLResolver2 instance used for name resolution when ENABLE_RESOLVER2 is set.
     * Unlike TSQLResolver which is created as a local variable, TSQLResolver2 is stored
     * as a property so users can retrieve name resolution results after parsing.
     *
     * Usage:
     * <pre>
     * TBaseType.setEnableResolver(false);  // Disable old resolver
     * TBaseType.setEnableResolver2(true);  // Enable new resolver
     *
     * TGSqlParser parser = new TGSqlParser(EDbVendor.dbvoracle);
     * parser.sqltext = sqlContent;
     * int ret = parser.parse();
     *
     * // Access resolver2 results
     * TSQLResolver2 resolver2 = parser.getResolver2();
     * if (resolver2 != null) {
     *     // Use TSQLResolver2ResultFormatter to generate output
     *     TSQLResolver2ResultFormatter formatter = new TSQLResolver2ResultFormatter(resolver2, config);
     *     String result = formatter.format();
     * }
     * </pre>
     */
    private TSQLResolver2 resolver2;

    /**
     * Get the TSQLResolver2 instance used for name resolution.
     * Returns null if resolver2 was not used or if parsing failed.
     *
     * @return the TSQLResolver2 instance or null
     */
    public TSQLResolver2 getResolver2() {
        return resolver2;
    }

    /**
     * The resolver type to use for name resolution.
     * Default is EResolverType.DEFAULT which uses TBaseType settings.
     */
    private EResolverType resolverType = EResolverType.DEFAULT;

    /**
     * Get the resolver type used for name resolution.
     *
     * @return the resolver type
     */
    public EResolverType getResolverType() {
        return resolverType;
    }

    /**
     * Set the resolver type to use for name resolution.
     *
     * <p>This instance-level setting takes precedence over global TBaseType settings.
     * When set to DEFAULT (the default value), behavior is determined by
     * TBaseType.isEnableResolver() and TBaseType.isEnableResolver2().</p>
     *
     * <h3>Usage Example:</h3>
     * <pre>
     * TGSqlParser parser = new TGSqlParser(EDbVendor.dbvoracle);
     * parser.setResolverType(EResolverType.RESOLVER2);  // Use new resolver
     * parser.sqltext = "SELECT * FROM employees";
     * parser.parse();
     *
     * // Access resolver2 results
     * TSQLResolver2 resolver = parser.getResolver2();
     * </pre>
     *
     * @param resolverType the resolver type to use
     * @see EResolverType
     */
    public void setResolverType(EResolverType resolverType) {
        this.resolverType = resolverType != null ? resolverType : EResolverType.DEFAULT;
    }

    /**
     * Optional configuration for TSQLResolver2.
     * If null, a default configuration will be created during parsing.
     */
    private TSQLResolverConfig resolver2Config;

    /**
     * Get the TSQLResolverConfig used for resolver2.
     * Returns null if not explicitly set (default config will be used during parsing).
     *
     * @return the resolver2 config or null
     */
    public TSQLResolverConfig getResolver2Config() {
        return resolver2Config;
    }

    /**
     * Set the TSQLResolverConfig to use for resolver2.
     *
     * <p>This allows customizing resolver2 behavior such as:</p>
     * <ul>
     *   <li>guessColumnStrategy - how to handle ambiguous columns</li>
     *   <li>legacyCompatibilityEnabled - sync results to legacy structures</li>
     *   <li>maxIterations - maximum iterations for iterative resolution</li>
     * </ul>
     *
     * <p>If not set, a default configuration will be created during parsing
     * with the database vendor automatically set.</p>
     *
     * <h3>Usage Example:</h3>
     * <pre>
     * TGSqlParser parser = new TGSqlParser(EDbVendor.dbvoracle);
     * parser.setResolverType(EResolverType.RESOLVER2);
     *
     * // Configure resolver2 to not pick ambiguous columns
     * TSQLResolverConfig config = new TSQLResolverConfig();
     * config.setGuessColumnStrategy(TSQLResolverConfig.GUESS_COLUMN_STRATEGY_NOT_PICKUP);
     * parser.setResolver2Config(config);
     *
     * parser.sqltext = "SELECT id FROM users, orders";
     * parser.parse();
     * </pre>
     *
     * @param config the resolver2 configuration, or null for default
     * @see TSQLResolverConfig
     */
    public void setResolver2Config(TSQLResolverConfig config) {
        this.resolver2Config = config;
    }

    /**
     * The array of syntax error generated by the parser during checking the syntax of the input SQL,
     * element of this list is type of {@link TSyntaxError}
     *
     * @return the array of errors
     */
    public ArrayList <TSyntaxError> getSyntaxErrors() {
        return syntaxErrors;
    }

    private ArrayList <TSyntaxError> syntaxErrors;

//    public ArrayList<TSyntaxError> getSyntaxHints() {
//        return syntaxHints;
//    }

    //private ArrayList <TSyntaxError> syntaxHints;

    /**
     * The database vendor specified when creating this parser.
     * The grammar rule of this database vendor will be used to validate the syntax of the input SQL.
     *
     * @return the database vendor
     */
    public EDbVendor getDbVendor() {
        return dbVendor;
    }

    /**
     * @deprecated As of v1.4.3.4
     * enable GSP to parse the rest of sql statements inside stored procedure
     * when a SQL statement in the stored procedure cannot be parsed
     *
     * <p>Available to parse sybase stored procedure currently.
     *
     * @param enablePartialParsing set true to enable this partial parsing, default is false
     */
    public void setEnablePartialParsing(boolean enablePartialParsing) {
        this.enablePartialParsing = enablePartialParsing;
    }

    /**
     * enable GSP to parse the rest of sql statements inside stored procedure
     * when a SQL statement in the stored procedure cannot be parsed
     *
     * <p>Available to parse sybase stored procedure currently.
     *
     * <p> default is false;
     *
     *  @deprecated As of v1.4.3.4
     */
    private boolean isEnablePartialParsing() {

        return enablePartialParsing;
    }

    private boolean enablePartialParsing = false;

    private boolean isSinglePLBlock = false;

    public void setSinglePLBlock(boolean singlePLBlock) {
        isSinglePLBlock = singlePLBlock;
    }

    private static String userName;
    private static String machineId = null;
    private static String licenseKey;
    private static String licenseType;
    private static boolean licenseOK = false;
    private static String licenseMessage;

    /**
     * Not used.
     *
     * @return the user name
     */
    public static String getUserName() {
        return userName;
    }

    /**
     * Not used.
     *
     * @return the machine id
     */
    public static String getMachineId() {

        return machineId;
    }

    /**
     * Not used.
     *
     * @return the license message
     */
    public static String getLicenseMessage() {
        return licenseMessage;
    }


    static {
        licenseOK = validateLicense();
    }

    /**
     * Not used.
     *
     * @return trial license or developer license or distribution license
     */
    public static String getLicenseType() {
        return licenseType;
    }

    private EDbVendor dbVendor;
    private String errormessage;

    /**
     * The lexer which is used to tokenize the input SQL.
     * For delegated vendors (MSSQL), lazily creates the vendor parser to get its lexer.
     *
     * @return the lexer
     */
    public TCustomLexer getFlexer() {
        if (flexer == null) {
            // Lazily create vendor parser to get its lexer
            SqlParser vp = getOrCreateVendorParser();
            if (vp instanceof gudusoft.gsqlparser.parser.MssqlSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.MssqlSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.MySqlSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.MySqlSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.PostgreSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.PostgreSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.OracleSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.OracleSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.BigQuerySqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.BigQuerySqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.AthenaSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.AthenaSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.CouchbaseSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.CouchbaseSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.DatabricksSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.DatabricksSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.DaxSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.DaxSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.Db2SqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.Db2SqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.GaussDbSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.GaussDbSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.GreenplumSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.GreenplumSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.HiveSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.HiveSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.HanaSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.HanaSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.ImpalaSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.ImpalaSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.InformixSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.InformixSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.MdxSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.MdxSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.NetezzaSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.NetezzaSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.OdbcSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.OdbcSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.OpenEdgeSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.OpenEdgeSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.PrestoSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.PrestoSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.RedshiftSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.RedshiftSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.SnowflakeSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.SnowflakeSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.SoqlSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.SoqlSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.SparksqlSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.SparksqlSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.SybaseSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.SybaseSqlParser) vp).flexer;
            } else if (vp instanceof gudusoft.gsqlparser.parser.VerticaSqlParser) {
                // Cache the flexer from vendor parser
                flexer = ((gudusoft.gsqlparser.parser.VerticaSqlParser) vp).flexer;
            }
        }
        return flexer;
    }

    private TCustomLexer flexer;

    TCustomParser fparser,fplsqlparser;

    // Cached vendor-specific parser for delegated parsing (MSSQL, etc.)
    // Created lazily in getFlexer() and reused in parse()
    private SqlParser vendorParser;

    BufferedReader finputstream = null; //used by lexer
    TCustomSqlStatement gcurrentsqlstatement,nextStmt;
    // Vendor-specific SQL command resolver
    ISqlCmds sqlcmds;

    HashMap sqlpluskeywordList;

    char delimiterchar;
    String defaultDelimiterStr;

    /**
     * Returns the delimiter character used to separate SQL statements.
     * Uses the flexer's delimiter if available (lexer-dependent), otherwise falls back to parser's value.
     * @return the delimiter character
     */
    public char getDelimiterChar() {
        if (flexer != null) {
            return flexer.delimiterchar;
        }
        return delimiterchar;
    }

    private ISQLStatementHandle sqlStatementHandle = null;

    public void setSqlStatementHandle(ISQLStatementHandle sqlStatementHandle) {
        this.sqlStatementHandle = sqlStatementHandle;
    }

    private ITokenHandle tokenHandle = null;

    /**
     * Set an event handler which will be fired when a new source token is created by the lexer during tokenize the
     * input SQL.
     *
     * @param tokenHandle the event handler to process the new created source token
     */
    public void setTokenHandle(ITokenHandle tokenHandle) {
        this.tokenHandle = tokenHandle;
    }

    private ITokenListHandle tokenListHandle = null;

    public void setTokenListHandle(ITokenListHandle tokenListHandle) {
        this.tokenListHandle = tokenListHandle;
    }

    private IMetaDatabase metaDatabase = null;

    /**
     * @deprecated As of v2.0.3.1, please use {@link #getSqlEnv()} instead
     *
     *  set an instance of a class which implement the interface: {@link IMetaDatabase}.
     *  The parser will call {@link IMetaDatabase#checkColumn} method when it needs to know
     *  whether a column is belonged to a table.
     *  <p></p>
     *  The class that implements the interface: {@link IMetaDatabase} usually fetch the metadata from the database
     *  by connecting to a database instance.
     *  <p></p>
     *  If the class is not provided, the parser has to guess the relationship between a un-qualified column and table
     *  in the input SQL which may lead to a un-determined result between the column and table.
     *
     * @param metaDatabase a new instance of the class which implements the {@link IMetaDatabase} interface
     * @see IMetaDatabase
     */
    public void setMetaDatabase(IMetaDatabase metaDatabase) {
        this.metaDatabase = metaDatabase;
    }

    /**
     * @deprecated As of v2.0.3.1, please use {@link #getSqlEnv()} instead
     *
     * a new instance of the class which implements the {@link IMetaDatabase} interface
     *
     * @return a new instance of the class which implements the {@link IMetaDatabase} interface
     * @see #setMetaDatabase
     */
    public IMetaDatabase getMetaDatabase() {

        return metaDatabase;
    }

    /**
     * Not used.
     *
     */
    public void freeParseTable() {
        flexer.yystack = null;
        flexer.yytextbuf = null;
        flexer.buf = null;

//        TLexerOracle.yyk = null;
//        TLexerOracle.yykl = null;
//        TLexerOracle.yykh = null;
//        TLexerOracle.yym = null;
//        TLexerOracle.yyml = null;
//        TLexerOracle.yymh = null;
//        TLexerOracle.yyt = null;
//        TLexerOracle.yytl = null;
//        TLexerOracle.yyth = null;
//        TParserOracleSql.yyah = null;
//        TParserOracleSql.yyal = null;
//        TParserOracleSql.yygh = null;
//        TParserOracleSql.yygl = null;
//        TParserOracleSql.yyd = null;
//        TParserOracleSql.yya_sym= null;
//        TParserOracleSql.yya_act= null;
//        TParserOracleSql.yyr_len= null;
//        TParserOracleSql.yyr_sym= null;
//        TParserOracleSql.yyg_sym= null;
//        TParserOracleSql.yyg_act= null;
//
//        TParserOraclePLSql.yyah = null;
//        TParserOraclePLSql.yyal = null;
//        TParserOraclePLSql.yygh = null;
//        TParserOraclePLSql.yygl = null;
//        TParserOraclePLSql.yyd = null;
//        TParserOraclePLSql.yya_sym= null;
//        TParserOraclePLSql.yya_act= null;
//        TParserOraclePLSql.yyr_len= null;
//        TParserOraclePLSql.yyr_sym= null;
//        TParserOraclePLSql.yyg_sym= null;
//        TParserOraclePLSql.yyg_act= null;
//
//        TLexerMssql.yyk = null;
//        TLexerMssql.yykl = null;
//        TLexerMssql.yykh = null;
//        TLexerMssql.yym = null;
//        TLexerMssql.yyml = null;
//        TLexerMssql.yymh = null;
//        TLexerMssql.yyt = null;
//        TLexerMssql.yytl = null;
//        TLexerMssql.yyth = null;
//        TParserMssqlSql.yyah = null;
//        TParserMssqlSql.yyal = null;
//        TParserMssqlSql.yygh = null;
//        TParserMssqlSql.yygl = null;
//        TParserMssqlSql.yyd = null;
//        TParserMssqlSql.yya_sym= null;
//        TParserMssqlSql.yya_act= null;
//        TParserMssqlSql.yyr_len= null;
//        TParserMssqlSql.yyr_sym= null;
//        TParserMssqlSql.yyg_sym= null;
//        TParserMssqlSql.yyg_act= null;
//
//        TLexerMysql.yyk = null;
//        TLexerMysql.yykl = null;
//        TLexerMysql.yykh = null;
//        TLexerMysql.yym = null;
//        TLexerMysql.yyml = null;
//        TLexerMysql.yymh = null;
//        TLexerMysql.yyt = null;
//        TLexerMysql.yytl = null;
//        TLexerMysql.yyth = null;
//        TParserMysqlSql.yyah = null;
//        TParserMysqlSql.yyal = null;
//        TParserMysqlSql.yygh = null;
//        TParserMysqlSql.yygl = null;
//        TParserMysqlSql.yyd = null;
//        TParserMysqlSql.yya_sym= null;
//        TParserMysqlSql.yya_act= null;
//        TParserMysqlSql.yyr_len= null;
//        TParserMysqlSql.yyr_sym= null;
//        TParserMysqlSql.yyg_sym= null;
//        TParserMysqlSql.yyg_act= null;
    }

    /**
     *  Class constructor, create a new instance of the parser by setting the database vendor
     *
     * @param pdbvendor the database vendor whose grammar rule will be used to validate the syntax of the input SQL
     */
    public TGSqlParser(EDbVendor pdbvendor) {
        dbVendor = pdbvendor;
        sqltext = "";
        sqlfilename = "";

        delimiterchar = ';';
        defaultDelimiterStr = ";";
        switch(pdbvendor){
            case dbvmssql:
            case dbvazuresql:
                // MSSQL and Azure SQL are delegated to MssqlSqlParser for tokenization
                // - flexer is accessed via getFlexer() which lazily creates MssqlSqlParser
                // - fparser is set in doDelegatedRawParse() from vendor parser result
                if (pdbvendor == EDbVendor.dbvazuresql) {
                    dbVendor = EDbVendor.dbvmssql;
                }
                break;
            case dbvaccess:
                // Access uses MSSQL lexer/parser but is NOT delegated
                if (TBaseType.enterprise_edition || (!TBaseType.full_edition) || TBaseType.sqlserver_edition || TBaseType.azuresql_edition) {
                    flexer = new TLexerMssql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMssqlSql(null);
                    fparser.lexer = flexer;
                }
                break;
            case dbvsybase:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sybase_edition){
                    flexer = new TLexerSybase();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSybase(null);
                    fparser.lexer = flexer;
                }
                enablePartialParsing = false;
                break;
            }
            case dbvinformix:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.informix_edition){
                    flexer = new TLexerInformix();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserInformix(null);
                    fparser.lexer = flexer;
                }
                enablePartialParsing = false;
                break;
            }
            case dbvoracle:
                // Oracle is delegated to OracleSqlParser for tokenization and parsing
                // - flexer is accessed via getFlexer() which lazily creates OracleSqlParser
                // - fparser and fplsqlparser are set in doDelegatedRawParse() from vendor parser result
                delimiterchar = '/';
                break;
            case dbvdb2 :{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.db2_edition){
                    delimiterchar = '@';
                    flexer = new TLexerDb2();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserDb2Sql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvmysql :{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.mysql_edition){
                    delimiterchar = '$';
                    defaultDelimiterStr = "$";
                    flexer = new TLexerMysql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMysqlSql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvteradata :{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.teradata_edition){
                    delimiterchar = '/';
                    flexer = new TLexerTeradata();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserTeradata(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvpostgresql:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.postgresql_edition){
                    delimiterchar = '/';
                    flexer = new TLexerPostgresql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserPostgresql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvredshift:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.redshift_edition){
                    delimiterchar = '/';
                    flexer = new TLexerRedshift();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserRedshift(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvgreenplum:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.greenplum_edition){
                    delimiterchar = '/';
                    flexer = new TLexerGreenplum();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserGreenplum(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvmdx:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.mdx_edition){
                    delimiterchar = ';';
                    flexer = new TLexerMdx();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMdx(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvnetezza:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.netezza_edition){
                    delimiterchar = ';';
                    flexer = new TLexerNetezza();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserNetezza(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvhive:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.hive_edition){
                    delimiterchar = ';';
                    flexer = new TLexerHive();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserHive(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvimpala:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.impala_edition){
                    delimiterchar = ';';
                    flexer = new TLexerImpala();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserImpala(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvhana:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.hana_edition){
                    delimiterchar = ';';
                    flexer = new TLexerHana();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserHana(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvdax:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.dax_edition){
                    delimiterchar = ';';
                    flexer = new TLexerDax();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserDax(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvodbc:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.odbc_edition){
                    delimiterchar = ';';
                    flexer = new TLexerOdbc();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserOdbc(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvvertica:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.vertica_edition){
                    delimiterchar = ';';
                    flexer = new TLexerVertica();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserVertica(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvopenedge:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.openedge_edition){
                    delimiterchar = ';';
                    flexer = new TLexerOpenedge();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserOpenedge(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvcouchbase:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.couchbase_edition){
                    delimiterchar = ';';
                    flexer = new TLexerCouchbase();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserCouchbase(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvsnowflake:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.snowflake_edition){
                    delimiterchar = ';';
                    flexer = new TLexerSnowflake();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSnowflake(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvbigquery:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.bigquery_edition){
                    delimiterchar = ';';
                    flexer = new TLexerBigquery();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserBigquery(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvclickhouse:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.clickhouse_edition){
                    delimiterchar = '/';
                    flexer = new TLexerClickhouse();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserClickhouse(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvansi:
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.ansi_edition){
                    delimiterchar = ';';
                    flexer = new TLexerAnsi();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserAnsi(null);
                    fparser.lexer = flexer;
                }
                break;
            case dbvgeneric: {
                if (TBaseType.enterprise_edition || (!TBaseType.full_edition) || TBaseType.generic_edition) {
                    flexer = new TLexerMssql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMssqlSql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvsoql:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.soql_edition){
                    delimiterchar = ';';
                    flexer = new TLexerSoql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSoql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvsparksql:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sparksql_edition){
                    delimiterchar = ';';
                    flexer = new TLexerSparksql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSparksql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvathena:
                {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.athena_edition){
                    delimiterchar = ';';
                    flexer = new TLexerathena();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserAthena(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvpresto:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.presto_edition){
                    delimiterchar = ';';
                    flexer = new TLexerPresto();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserPresto(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvtrino:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.trino_edition){
                    delimiterchar = ';';
                    flexer = new TLexerPresto();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserPresto(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvdatabricks:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.databricks_edition){
                    delimiterchar = ';';
                    flexer = new TLexerDatabricks();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserDatabricks(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvgaussdb:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.gaussdb_edition){
                    delimiterchar = ';';
                    flexer = new TLexerGaussDB();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserGaussDB(null);
                    fparser.lexer = flexer;
                   // fplsqlparser = new TParserOraclePLSql(null);
                   // fplsqlparser.lexer = new TLexerOracle();// must use oracle lexer, not gauss lexer
                }
                break;
            }
            default:  {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sqlserver_edition){
                    flexer = new TLexerMssql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMssqlSql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
        }

        // For delegated vendors (e.g., MSSQL), fparser may be null here
        // It will be set later in doDelegatedRawParse() from vendor parser result
        if (fparser != null) {
            fparser.getNf().setGsqlParser(this);
        }

        sourcetokenlist = new TSourceTokenList();
        sourcetokenlist.setGsqlparser(this);
        sqlstatements = new TStatementList();
        // Inject vendor-specific command resolver
        sqlcmds = SqlCmdsFactory.get(dbVendor);
        if (fparser != null) {
            fparser.sqlcmds = sqlcmds;
        }
        sqlpluskeywordList = new HashMap();
        syntaxErrors = new ArrayList();
        //syntaxHints = new ArrayList();


        errormessage = "";

        if (TBaseType.license_expired_check){
            if (!check_license_time()) {
                flexer = null;
                fparser = null;
               // System.out.println("license is expired");
            }
        }

        TGSqlParser.currentDBVendor = dbVendor;
    }

    TContext globalContext;


    /**
     * Create a select statement object from the parameter: subquery
     *
     * @param subquery a plain text select statement which need to be converted to a {@link gudusoft.gsqlparser.stmt.TSelectSqlStatement} object
     * @return a select statement object, return null if the input select string is syntax invalid.
     */
    public TSelectSqlStatement parseSubquery(String subquery){
        return parseSubquery(this.dbVendor,subquery);
    }

    /**
     * this method is thread safe.
     * 
     * C:\prg\gsp_java\gsp_java_core\src\test\java\gudusoft\gsqlparser\ParseSubqueryThreadSafetyTest.java
     * 
     */
    public static TSelectSqlStatement parseSubquery(EDbVendor dbVendor, String subquery){
//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = subquery;
//        int iRet = localParser.doparse();
//        if (iRet != 0) {
//            return null;
//        }

        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,subquery);
        if (statements.size() == 0) return null;

        return (TSelectSqlStatement)statements.get(0);
    }

    /**
     * Create an expression object from the parameter: expr
     *
     * @param expr a plain text expression which will be converted to {@link gudusoft.gsqlparser.nodes.TExpression} object
     * @return an expression object, return null if the input expression string is not syntax valid.
     */
    public TExpression parseExpression(String expr){
        return parseExpression(this.dbVendor,expr);
    }

    /*
     * this method is thread safe.
     * 
     * getStmts() is synchronized - only one thread can execute it at a time
     * getParser() is also synchronized
     * Each thread gets a new TStatementList instance

     * this is the test case for this method.
     * C:\prg\gsp_java\gsp_java_core\src\test\java\gudusoft\gsqlparser\ParseExpressionThreadSafetyTest.java
     */
    public static TExpression parseExpression(EDbVendor dbVendor, String expr){
        TSingletonParser singletonParser = TSingletonParser.getInstance();

        boolean e = TBaseType.isEnableResolver();
        TBaseType.setEnableResolver(false);
        TStatementList statements;
        try{
            statements  = singletonParser.getStmts(dbVendor,"select 1 from t where "+TBaseType.newline+expr);
        }finally {
            TBaseType.setEnableResolver(e);
        }

        if (statements.size() == 0) return null;

//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select 1 from t where "+TBaseType.newline+expr;
//        int iRet = localParser.doparse();
//        if (iRet != 0) {
//            return null;
//        }

        return ((TSelectSqlStatement)statements.get(0)).getWhereClause().getCondition();
    }

    /**
     *  Create a function object from the parameter: newFunction
     *
     * @param newFunction a plain text function which will be converted to {@link gudusoft.gsqlparser.nodes.TFunctionCall} object
     * @return a function object, or return null if the input string is not a valid function call.
     */
    public TFunctionCall parseFunctionCall(String newFunction){
        return parseFunctionCall(this.dbVendor,newFunction);
    }

    public static TFunctionCall parseFunctionCall(EDbVendor dbVendor, String newFunction){
        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,"select"+TBaseType.newline+newFunction+TBaseType.newline+"from t");
        if (statements.size() == 0) return null;


//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select"+TBaseType.newline+newFunction+TBaseType.newline+"from t";
//        int iRet = localParser.doparse();
//        if (iRet!= 0) {
//            return null;
//        }

        return statements.get(0).getResultColumnList().getResultColumn(0).getExpr().getFunctionCall();
    }

    /**
     * Create a database objectName from the parameter: newObjectName
     *
     * @param newObjectName a plain text objectName which will be converted to {@link gudusoft.gsqlparser.nodes.TObjectName} object
     * @return a database objectName object, return null is the input string is not a valid objectName.
     */
    public TObjectName parseObjectName(String newObjectName){
        //return parseObjectName(this.dbVendor,newObjectName);
        return TObjectName.createObjectName(this.dbVendor,EDbObjectType.column,newObjectName);
    }



    /**
     *
     * @param dbVendor
     * @param newObjectName
     * @return
     */
    public static TObjectName parseObjectName(EDbVendor dbVendor, String newObjectName){

        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,"select"+TBaseType.newline+newObjectName+TBaseType.newline+"from t");
        if (statements.size() == 0) return null;
        TExpression e = statements.get(0).getResultColumnList().getResultColumn(0).getExpr();

//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select"+TBaseType.newline+newObjectName+TBaseType.newline+"from t";
//        int iRet = localParser.doparse();
//        if (iRet!= 0) {
//            return null;
//        }
//        TExpression e = ((TSelectSqlStatement)localParser.sqlstatements.get(0)).getResultColumnList().getResultColumn(0).getExpr();

        TObjectName lcResult = null;
        switch (e.getExpressionType()){
            case simple_object_name_t:
                lcResult = e.getObjectOperand();
                break;
            case simple_constant_t:
                lcResult = new TObjectName();
                lcResult.init(e.getConstantOperand().getValueToken());
                break;
            default:
                break;
        }

        return lcResult;
    }

    /**
     * Create an constant object from the parameter: newConstant
     *
     * @param newConstant a plian text constant which will be converted to {@link gudusoft.gsqlparser.nodes.TConstant} object
     * @return new constant object, returns null if the input is not a valid constant string.
     */
    public TConstant parseConstant(String newConstant){
        return parseConstant(this.dbVendor,newConstant);
    }

    public static TConstant parseConstant(EDbVendor dbVendor, String newConstant){
//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select"+TBaseType.newline+newConstant+TBaseType.newline+"from t";
//        int iRet = localParser.doparse();
//        if (iRet!= 0) {
//            return null;
//        }

        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,"select"+TBaseType.newline+newConstant+TBaseType.newline+"from t");
        if (statements.size() == 0) return null;

        return statements.get(0).getResultColumnList().getResultColumn(0).getExpr().getConstantOperand();
    }


    /**
     * The total number of syntax errors founded in the input SQL script.
     * <br>Only the first syntax error in a SQL statement is counted if there are more than one syntax errors in
     * a single SQL statement.
     * <br> In a SQL script, only the first syntax error in each SQL statement is counted.
     *
     * @return The total number of syntax errors founded in the input SQL script. Returns 0 if no syntax error is founded.
     */
    public int getErrorCount(){
        return   syntaxErrors.size();
    }

    /**
     * The text of error message generated by iterating all items in {@link #getSyntaxErrors}.
     * User may generate error message in their own format by iterating all items in {@link #getSyntaxErrors}.
     *
     * @return error message
     */
    public String getErrormessage(){

        String s="",hint="Syntax error";
        TSyntaxError t;
        for (int i= 0; i< syntaxErrors.size(); i++)
        {
            t = (TSyntaxError) syntaxErrors.get(i);
            if (t.hint.length() > 0) hint = t.hint;
            s= s+hint+"("+t.errorno+") near: "+t.tokentext;
            s=s+"("+t.lineNo;
            s=s+","+t.columnNo +", token code:"+t.tokencode+")";
                //s=s+" expected tokentext:"+t.hint;

           // break;//get only one message, remove this one and uncomment next line to get all error messages
            if (i !=  syntaxErrors.size() - 1)
              s = s +TBaseType.linebreak;
        }

        if (errormessage.length() > 0){
            s = errormessage+TBaseType.linebreak+s; 
        }
        return s;

    }

    /**
     * check syntax of the input SQL. This method works exactly the same as {@link #parse} method.
     *
     * @return   0 means parse SQL script successfully, otherwise, use {@link #getErrorCount}, {@link #getErrormessage}
     * to get detailed error information.
     * @see #parse
     */
    public int checkSyntax(){
        return doparse();
    }

    /**
     *   Check syntax of the input SQL, doing some kind of semantic analysis without connecting to a real database.
     *   <p></p>
     *  This method will do a in-depth analysis of the input SQL such as building the link between table and columns.
     *  The parse tree of the input SQL is available after calling this method.
     *
     *  The parser checks the syntax of those SQL statements one by one. If syntax error is found in a SQL statement,
     *  an error will be logged, no parse tree will be built for this SQL statement,
     *  the error message can be fetched using the {@link #getErrormessage()} method.
     *  <p></p>
     *  The syntax error in one SQL statement doesn't prevent the parser continue to check the syntax of the next SQL statement.
     *  After checking syntax of all SQL statements, use the {@link #getErrorCount()} method to get the total number of errors.
     *  <p></p>
     *  A syntax error in a SQL stored procedure will cease this parser to check syntax of the rest SQL statements
     *  in this stored procedure.
     *
     * @return   0 means parse SQL script successfully, otherwise, use {@link #getErrorCount}, {@link #getErrormessage}
     * to get detailed error information.
     * @see #getSyntaxErrors
     */

    public int parse(){
        return doparse();
    }

    public int validate(){
        if (sqlstatements.size() == 0) return 0;

        TRelationValidator relationValidate = new TRelationValidator(globalContext);
        for(TCustomSqlStatement sqlStatement:sqlstatements){
            sqlStatement.acceptChildren(relationValidate);
        }
        return 0;
    }

    void setdelimiterchar(char ch){
        delimiterchar = ch;
        flexer.delimiterchar = ch;
    }
    char curdelimiterchar;
    String userDelimiterStr ="";

    boolean includesqlstatementtype(ESqlStatementType search, ESqlStatementType[] src){
        boolean ret = false;
        for(int i=0;i<src.length;i++){
            if (src[i] == search){
                ret = true;
                break;
            }
        }
        return ret;
    }

    // private int getfileEncodingType(FileInputStream inputStream){
    //     BufferedInputStream fr = new BufferedInputStream(inputStream,8);
    //     return getfileEncodingType(fr);
    // }

    private int getfileEncodingType(BufferedInputStream fr){
        int ret = 0; // default, 1: utf-16, 2: utf-32
       // BufferedInputStream fr = new BufferedInputStream(inputStream,8);
        try {
            byte[] bom = new byte[4];
            fr.mark(bom.length+1);

            fr.read(bom,0,bom.length);
            if ( ((bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE))
                    ||((bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF))
                    )
            {
                ret = 1;
                if ( ((bom[2] == (byte)0xFF) && (bom[3] == (byte)0xFE))
                        ||((bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF))
                        ){
                    ret = 2;
                }
            }else{
                if ((bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB)&& (bom[2] == (byte)0xBF)){
                    ret = 3; //UTF-8,EF BB BF
                }
            }
            fr.reset();
        } catch (FileNotFoundException e) {
            // e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
        } catch (IOException e) {
            //e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
        }
        return ret;
    }

    // private int getfileEncodingType(String fn){
    //     int ret = 0; // default, 1: utf-16, 2: utf-32
    //     try {
    //         FileInputStream fr =   new FileInputStream(fn);
    //         ret = getfileEncodingType(fr);
    //         fr.close();

    //     } catch (FileNotFoundException e) {
    //        // e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    //     } catch (IOException e) {
    //         //e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    //     }
    //     return ret;
    // }

    int readsql(){
        int ret  = 0;

        syntaxErrors.clear();
        //syntaxHints.clear();

        if (((!TBaseType.full_edition))||(!TBaseType.need_license_file)){
            if (!TBaseType.need_license_file){
                licenseType = TBaseType.license_type_dist;
                userName = "dist";
            }else {
                licenseType = TBaseType.license_type_trial;
                userName = TBaseType.license_trail_username;
            }
           // machineId = new HardwareBinder().getMachineIdString();
        }else {
            if (!licenseOK){
                errormessage = licenseMessage;
                return -1;
            }
        }

        try{
            if (finputstream != null) finputstream.close();
            if (flexer == null){
                ret = -1;
                errormessage = "requested database not supported:"+this.dbVendor.toString();
                return ret;
            }
            if (flexer.yyinput != null)    flexer.yyinput.close();

        }catch(IOException e){
            ret = -1;
            errormessage = "requested database not supported";
        }
        
        if (sqltext.length() > 0){
            finputstream = new BufferedReader(new StringReader(sqltext), TBaseType.LEXER_INPUT_BUFFER_SIZE);
            if ((!TBaseType.full_edition) && (sqltext.length() > TBaseType.query_size_limitation)){
                errormessage = TBaseType.trail_version_query_message;
                ret = -1;
            }
        }else if (sqlfilename.length() > 0){
            try{

             streamFromSqlFile =   new FileInputStream(sqlfilename);
            // Buffer it for mark/reset support
            BufferedInputStream bufferedStream = new BufferedInputStream(streamFromSqlFile,8);             
            int encodingtype = getfileEncodingType(bufferedStream);
            streamFromSqlFile.getChannel().position(0);

           if(encodingtype == 1){
                sqlStreamReader = new InputStreamReader(streamFromSqlFile,"UTF-16");
           }else if(encodingtype == 2){
                sqlStreamReader = new InputStreamReader(streamFromSqlFile,"UTF-32");
           }else if(encodingtype == 3){
              // System.out.println("utf-8");
                sqlStreamReader = new InputStreamReader(streamFromSqlFile,"UTF-8");
           }
           else{
               if (sqlCharset == null){
                   sqlCharset = Charset.defaultCharset().name();
                   //System.out.println("Charset used: "+Charset.defaultCharset().name());
               }
                sqlStreamReader = new InputStreamReader(streamFromSqlFile, sqlCharset);
              // isr = new InputStreamReader(fr,"Cp737");
           }

            finputstream =  new BufferedReader(sqlStreamReader, TBaseType.LEXER_INPUT_BUFFER_SIZE);
            if (encodingtype == 3){
                //EF BB BF was not stripped by the InputStreamReader, so we do it
                finputstream.skip(1);
            }

            if ((!TBaseType.full_edition)){
                File file = new File(sqlfilename);
                if (!file.exists() || !file.isFile()) {
                    ret = -1;
                    errormessage = "not a valid sql file.";
                }else{
                    if (file.length() > TBaseType.query_size_limitation){
                        errormessage = TBaseType.trail_version_file_message;
                        ret = -1;
                    }
                }
            }

            }catch(FileNotFoundException e){
                ret = -1;
                errormessage = e.toString();
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            } catch (IOException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }

        }else if(this.sqlInputStream != null){
            int encodingtype = getfileEncodingType(sqlInputStream);
            InputStream fr =   sqlInputStream;

            InputStreamReader isr = null;
            try{

                if(encodingtype == 1){
                    isr = new InputStreamReader(fr,"UTF-16");
                }else if(encodingtype == 2){
                    isr = new InputStreamReader(fr,"UTF-32");
                }else if(encodingtype == 3){
                    // System.out.println("utf-8");
                    isr = new InputStreamReader(fr,"UTF-8");
                }
                else{
                    if (sqlCharset == null){
                        sqlCharset = Charset.defaultCharset().name();
                    }

                    isr = new InputStreamReader(fr, sqlCharset);
                }

                finputstream =  new BufferedReader(isr, TBaseType.LEXER_INPUT_BUFFER_SIZE);
                if (encodingtype == 3){
                    //EF BB BF was not stripped by the InputStreamReader, so we do it
                    finputstream.skip(1);
                }

            }catch(FileNotFoundException e){
                ret = -1;
                errormessage = e.toString();
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            } catch (IOException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }

            try {
                if ((!TBaseType.full_edition) && (sqlInputStream.available() > TBaseType.query_size_limitation)){
                    errormessage = TBaseType.trail_version_query_message;
                    ret = -1;
                }
            } catch (IOException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }
        }

       if (finputstream == null) ret = -1;

       if (ret == 0)
       {
           flexer.yyinput =  finputstream;
           flexer.setSqlCharset(this.sqlCharset);
           flexer.reset();
       }

        sourcetokenlist.clear();
        sourcetokenlist.curpos = -1;

       flexer.resetTokenTable();


        return ret;
    }

    TSourceToken getanewsourcetoken(){
        TSourceToken pst = null,prevst;
        
        while (true) {
            pst = new TSourceToken("");
            if (flexer.yylexwrap(pst) == 0) { pst = null; break;}
            
            pst.setDbvendor(dbVendor);
            pst.tokenstatus = ETokenStatus.tsoriginal;
            if (pst.tokentype == ETokenType.ttreturn){
               pst.setAstext(towinlinebreak(pst.getAstext()));
            }
            //combine space && linebreak after a linebreak into one
            if ( (pst.tokentype == ETokenType.ttwhitespace)
             && (sourcetokenlist.curpos >= 0) )
              {
                prevst =  sourcetokenlist.get(sourcetokenlist.curpos);
                if ( prevst.tokentype == ETokenType.ttreturn )
                  {
                    //can't discard  whitespace after linebreak, it will be used
                    // to judge whether / at the { of the line is a sqlplus cmd or not
                    // check isvalidplacefordivtosqlpluscmd for more
                    prevst.setAstext(prevst.getAstext() + pst.getAstext());
                    //newst.free;
                    //newst = nil;
                    continue;
                  }
              }

            if ( (pst.tokentype == ETokenType.ttreturn)
             && (sourcetokenlist.curpos >= 0) )
              {
                prevst =  sourcetokenlist.get(sourcetokenlist.curpos);

                if ( prevst.tokentype == ETokenType.ttreturn )
                  {
                    prevst.setAstext(prevst.getAstext() + pst.getAstext());
                    //newst.free;
                    //newst = nil;
                    continue;
                  }

                if ( prevst.tokentype == ETokenType.ttwhitespace )
                  {
                    //merge previous whitespace with this linebreak
                    //doesn't work for sqlplus cmd

                   // prevst.sourcecode = newst.sourcecode;
                   // prevst.tokentype = newst.tokentype;
                   // prevst.tokencode = newst.tokencode;
                   // newst.free;
                   // newst = nil;
                   // continue;

                   //prevst.prevsourcecode = prevst.astext;
                   //prevst.astext =  "";
                  }
              }

            break;

        }

        if (pst != null){
            pst.container = sourcetokenlist;
            sourcetokenlist.curpos = sourcetokenlist.curpos+1;
            pst.posinlist = sourcetokenlist.curpos;
            if (tokenHandle != null){
                tokenHandle.processToken(pst);
            }
        }

       // System.out.println(pst);
      //  flexer.setTokenTableValue(pst);
        return pst;
        
    }

  String towinlinebreak(String s){
      return s;
// todo not implemented yet     
  }

void checkconstarinttoken(TSourceToken lcprevtoken){
    TSourceTokenList lcStList = lcprevtoken.container;
    if (TBaseType.assigned(lcStList))
    {
         TSourceToken lcPPToken = lcStList.nextsolidtoken(lcprevtoken.posinlist,-2,false);
         if (TBaseType.assigned(lcPPToken))
        {

             if (lcPPToken.tokencode == flexer.getkeywordvalue("constraint"))
             {
                 //System.out.println(lcPPToken);
                 lcPPToken.tokencode = TBaseType.rw_constraint2;
             }
        }
    }

}

TSourceToken getprevtoken(TSourceToken ptoken){
   // ETokenType[] lcnonsolidtokenset = {ETokenType.ttwhitespace,ETokenType.ttreturn,ETokenType.ttsimplecomment,ETokenType.ttbracketedcomment} ;
    TSourceTokenList lcstlist = ptoken.container;
    if (TBaseType.assigned(lcstlist)){
        if ((ptoken.posinlist > 0)  && (lcstlist.size() > ptoken.posinlist-1))
        {
            if (!(
                    (lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttwhitespace)
                    ||(lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttreturn)
                    ||(lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttsimplecomment)
                    ||(lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttbracketedcomment)
                    )
                )
            {return lcstlist.get(ptoken.posinlist-1);}
            else{
            { return lcstlist.nextsolidtoken(ptoken.posinlist-1,-1,false);}
        }
        }
    }
    
    return null;
}

void doinformixtexttotokenlist(){
      TSourceToken lcprevtoken = null;
      int lcsteps = 0;
      TSourceToken asourcetoken,lctoken, lctoken2,lctoken3;
      int yychar;
      boolean iskeywordgo;

      asourcetoken = getanewsourcetoken();

      if ( asourcetoken == null ) return;

      yychar = asourcetoken.tokencode;


      boolean lcinopenrowset = false;
      int lcnested = 0;

      while (yychar > 0)
        {

          if ( asourcetoken.tokencode == TBaseType.rrw_openrowset )
            {
              // openrowset(....)
              lcinopenrowset = true;
              lcnested = 0;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttleftparenthesis)
            {
              if ( (lcsteps > 0) && TBaseType.assigned(lcprevtoken))
                {
                  if ( lcprevtoken.tokencode == TBaseType.rrw_primary )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 2;  //rw_primary2
                     //checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_foreign )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 4;  //rw_foreign2
                     //checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_unique )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 1;  //rw_unique2
                     // checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_distinct )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 6;  //rw_distinct2
                      //checkconstarinttoken(lcprevtoken);
                    }
                  lcprevtoken = null;
                  lcsteps = 0;
                }

              // openrowset(....)
              if ( lcinopenrowset )
                lcnested++;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttrightparenthesis)
            {
              // openrowset(....)
              if ( lcinopenrowset  )
                {
                  if ( (lcnested > 0) )
                    lcnested--;
                  if ( lcnested == 0 )
                    lcinopenrowset = false;
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttsemicolon )
            {
              if ( lcinopenrowset ){
                asourcetoken.tokentype = ETokenType.ttsemicolon2;
              }else{
                lctoken2 = asourcetoken.searchToken(TBaseType.rrw_begin,-1);
                  if (lctoken2 != null){
                     asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                     asourcetoken.tokentype = ETokenType.ttsemicolon3;
                  }else{
                      lctoken2 = asourcetoken.searchToken(TBaseType.rrw_try,-1);
                      if (lctoken2 == null){
                          lctoken2 = asourcetoken.searchToken(TBaseType.rrw_catch,-1);
                      }
                      if (lctoken2 != null){
                          lctoken3 = asourcetoken.searchToken(TBaseType.rrw_begin,-2);
                          if (lctoken3 != null){
                              asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                              asourcetoken.tokentype = ETokenType.ttsemicolon3;
                          }
                      }
                  }
              }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttperiod)
            {
              lctoken = getprevtoken(asourcetoken);
             //  System.out.println(lctoken);
              if ( TBaseType.assigned(lctoken) )
                { // go.fieldname, go is a table alias, not a go statement
                  if ( lctoken.tokencode == TBaseType.rrw_go )
                    {
                      lctoken.tokencode = TBaseType.ident;
                      lctoken.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_table ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_lock )
                  {
                      lctoken.tokencode = TBaseType.rw_locktable; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_to ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_connect )
                  {
                      lctoken.tokencode = TBaseType.rrw_informix_connect_to;//connect to statement
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_go )
            {
              iskeywordgo = true;
              lctoken = getprevtoken(asourcetoken);
              if ( TBaseType.assigned(lctoken) )
                { // go should not at same line as other sql statement
                  if ( lctoken.lineNo == asourcetoken.lineNo)
                    {
                      iskeywordgo = false;
                    }
                }

              if ( iskeywordgo )
                {
                  lcinopenrowset = false;
                  lcnested = 0;
                  lcprevtoken = asourcetoken;
                }
              else
                {
                  //  System.out.println(asourcetoken);
                  asourcetoken.tokencode = TBaseType.ident;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_primary )
            {
              // primary key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_foreign )
            {
              // foreign key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_unique )
            {
              // unique [clustered | nonclustered ] ( column list)
              lcsteps = 1;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.issolidtoken() )
            {
              if ( lcsteps > 0 )
                {
                  if (  !(TBaseType.mysametext("clustered",asourcetoken.toString())
                     || TBaseType.mysametext("nonclustered",asourcetoken.toString())) )
                    lcsteps--;
                }
            }

           //System.out.println(asourcetoken);
           sourcetokenlist.add(asourcetoken);


          //flexer.yylexwrap(asourcetoken);
          asourcetoken = getanewsourcetoken();
          if (asourcetoken != null){
              yychar = asourcetoken.tokencode;
          }else{
            yychar = 0;
          }

        }

  }

  void domssqlsqltexttotokenlist(){
      TSourceToken lcprevtoken = null;
      int lcsteps = 0;
      TSourceToken asourcetoken,lctoken, lctoken2,lctoken3;
      int yychar;
      boolean iskeywordgo;

      asourcetoken = getanewsourcetoken();
      
      if ( asourcetoken == null ) return;

      yychar = asourcetoken.tokencode;


      boolean lcinopenrowset = false;
      int lcnested = 0;

      while (yychar > 0)
        {

          if ( asourcetoken.tokencode == TBaseType.rrw_openrowset )
            {
              // openrowset(....)
              lcinopenrowset = true;
              lcnested = 0;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttleftparenthesis)
            {
              if ( (lcsteps > 0) && TBaseType.assigned(lcprevtoken))
                {
                  if ( lcprevtoken.tokencode == TBaseType.rrw_primary )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 2;  //rw_primary2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_foreign )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 4;  //rw_foreign2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_unique )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 1;  //rw_unique2
                      checkconstarinttoken(lcprevtoken);
                    }
                  lcprevtoken = null;
                  lcsteps = 0;
                }

              // openrowset(....)
              if ( lcinopenrowset )
                lcnested++;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttrightparenthesis)
            {
              // openrowset(....)
              if ( lcinopenrowset  )
                {
                  if ( (lcnested > 0) )
                    lcnested--;
                  if ( lcnested == 0 )
                    lcinopenrowset = false;
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttsemicolon )
            {
              if ( lcinopenrowset ){
                asourcetoken.tokentype = ETokenType.ttsemicolon2;
              }else{
                lctoken2 = asourcetoken.searchToken(TBaseType.rrw_begin,-1);
                  if (lctoken2 != null){
                     asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                     asourcetoken.tokentype = ETokenType.ttsemicolon3;
                  }else{
                      lctoken2 = asourcetoken.searchToken(TBaseType.rrw_try,-1);
                      if (lctoken2 == null){
                          lctoken2 = asourcetoken.searchToken(TBaseType.rrw_catch,-1);
                      }
                      if (lctoken2 != null){
                          lctoken3 = asourcetoken.searchToken(TBaseType.rrw_begin,-2);
                          if (lctoken3 != null){
                              asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                              asourcetoken.tokentype = ETokenType.ttsemicolon3;
                          }
                      }
                  }
              }

                lctoken = getprevtoken(asourcetoken);
                if ((lctoken != null)){
                    if (lctoken.tokentype == ETokenType.ttsemicolon){
                        // treat this semicolon as a whitespace
                        asourcetoken.tokencode = TBaseType.lexspace;
                    }
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttperiod)
            {
              lctoken = getprevtoken(asourcetoken);
             //  System.out.println(lctoken);
              if ( TBaseType.assigned(lctoken) )
                { // go.fieldname, go is a table alias, not a go statement
                  if ( lctoken.tokencode == TBaseType.rrw_go )
                    {
                      lctoken.tokencode = TBaseType.ident;
                      lctoken.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_table ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_lock )
                  {
                      lctoken.tokencode = TBaseType.rw_locktable; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_into ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_sqlserver_copy )
                  {
                     lctoken.tokencode = TBaseType.rrw_sqlserver_copyinto;
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_sqlserver_column ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_drop )
                  {
                      asourcetoken.tokencode = TBaseType.rrw_sqlserver_drop_column;
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_go )
            {
              iskeywordgo = true;
              lctoken = getprevtoken(asourcetoken);
              if ( TBaseType.assigned(lctoken) )
                { // go should not at same line as other sql statement
                  if (( lctoken.lineNo == asourcetoken.lineNo)&&(lctoken.tokencode != ';'))
                    {
                      iskeywordgo = false;
                    }
                }

              if ( iskeywordgo )
                {
                  lcinopenrowset = false;
                  lcnested = 0;
                  lcprevtoken = asourcetoken;
                }
              else
                {
                  //  System.out.println(asourcetoken);
                  asourcetoken.tokencode = TBaseType.ident;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_primary )
            {
              // primary key [clustered | nonclustered ] [hash] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_foreign )
            {
              // foreign key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_unique )
            {
              // unique [clustered | nonclustered ] [hash] ( column list)
              lcsteps = 1;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.issolidtoken() )
            {
              if ( lcsteps > 0 )
                {
                  if (  !(TBaseType.mysametext("clustered",asourcetoken.toString())
                     || TBaseType.mysametext("nonclustered",asourcetoken.toString())
                     || TBaseType.mysametext("hash",asourcetoken.toString())
                  ) )
                    lcsteps--;
                }
            }

           //System.out.println(asourcetoken); 
           sourcetokenlist.add(asourcetoken);


          //flexer.yylexwrap(asourcetoken);
          asourcetoken = getanewsourcetoken();
          if (asourcetoken != null){
              yychar = asourcetoken.tokencode;
          }else{
            yychar = 0;
          }

        }

  }

  void dosybasesqltexttotokenlist(){
      TSourceToken lcprevtoken = null;
      int lcsteps = 0;
      TSourceToken asourcetoken,lctoken, lctoken2,lctoken3;
      int yychar;
      boolean iskeywordgo;


      asourcetoken = getanewsourcetoken();

      if ( asourcetoken == null ) return;

      yychar = asourcetoken.tokencode;


      boolean lcinopenrowset = false;
      int lcnested = 0;

      while (yychar > 0)
        {

          if ( asourcetoken.tokencode == TBaseType.rrw_openrowset )
            {
              // openrowset(....)
              lcinopenrowset = true;
              lcnested = 0;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttleftparenthesis)
            {
              if ( (lcsteps > 0) && TBaseType.assigned(lcprevtoken))
                {
                  if ( lcprevtoken.tokencode == TBaseType.rrw_primary )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 2;  //rw_primary2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_foreign )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 4;  //rw_foreign2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_unique )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 1;  //rw_unique2
                      checkconstarinttoken(lcprevtoken);
                    }
                  lcprevtoken = null;
                  lcsteps = 0;
                }

              // openrowset(....)
              if ( lcinopenrowset )
                lcnested++;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttrightparenthesis)
            {
              // openrowset(....)
              if ( lcinopenrowset  )
                {
                  if ( (lcnested > 0) )
                    lcnested--;
                  if ( lcnested == 0 )
                    lcinopenrowset = false;
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttsemicolon )
            {
              if ( lcinopenrowset ){
                asourcetoken.tokentype = ETokenType.ttsemicolon2;
              }else{
                lctoken2 = asourcetoken.searchToken(TBaseType.rrw_begin,-1);
                  if (lctoken2 != null){
                     asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                     asourcetoken.tokentype = ETokenType.ttsemicolon3;
                  }else{
                      lctoken2 = asourcetoken.searchToken(TBaseType.rrw_try,-1);
                      if (lctoken2 == null){
                          lctoken2 = asourcetoken.searchToken(TBaseType.rrw_catch,-1);
                      }
                      if (lctoken2 != null){
                          lctoken3 = asourcetoken.searchToken(TBaseType.rrw_begin,-2);
                          if (lctoken3 != null){
                              asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                              asourcetoken.tokentype = ETokenType.ttsemicolon3;
                          }
                      }
                  }
              }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttperiod)
            {
              lctoken = getprevtoken(asourcetoken);
             //  System.out.println(lctoken);
              if ( TBaseType.assigned(lctoken) )
                { // go.fieldname, go is a table alias, not a go statement
                  if ( lctoken.tokencode == TBaseType.rrw_go )
                    {
                      lctoken.tokencode = TBaseType.ident;
                      lctoken.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_table ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_lock )
                  {
                      lctoken.tokencode = TBaseType.rw_locktable; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_sybase_isolation ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_sybase_at )
                  {
                      lctoken.tokencode = TBaseType.rw_sybase_at1; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_update ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if ((lctoken.tokencode == TBaseType.rrw_not )
                          ||(lctoken.tokencode == TBaseType.rrw_and )
                          ||(lctoken.tokencode == TBaseType.rrw_or )
                          ||(lctoken.tokencode == TBaseType.rrw_if )
                      )
                  {
                      // this is update(column) in create trigger
                      asourcetoken.tokencode = TBaseType.rw_sybase_update1;
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_go )
            {
              iskeywordgo = true;
              lctoken = getprevtoken(asourcetoken);
              if ( TBaseType.assigned(lctoken) )
                { // go should not at same line as other sql statement
                  if ( lctoken.lineNo == asourcetoken.lineNo)
                    {
                      iskeywordgo = false;
                    }
                }

              if ( iskeywordgo )
                {
                  lcinopenrowset = false;
                  lcnested = 0;
                  lcprevtoken = asourcetoken;
                }
              else
                {
                  //  System.out.println(asourcetoken);
                  asourcetoken.tokencode = TBaseType.ident;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_primary )
            {
              // primary key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_foreign )
            {
              // foreign key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_unique )
            {
              // unique [clustered | nonclustered ] ( column list)
              lcsteps = 1;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.issolidtoken() )
            {
              if ( lcsteps > 0 )
                {
                  if (  !(TBaseType.mysametext("clustered",asourcetoken.toString())
                     || TBaseType.mysametext("nonclustered",asourcetoken.toString())) )
                    lcsteps--;
                }
            }

           //System.out.println(asourcetoken);
           sourcetokenlist.add(asourcetoken);


          //flexer.yylexwrap(asourcetoken);
          asourcetoken = getanewsourcetoken();
          if (asourcetoken != null){
              yychar = asourcetoken.tokencode;
          }else{
            yychar = 0;
          }

        }

  }

boolean IsValidPlaceForDivToSqlplusCmd(TSourceTokenList pstlist, int pPos){
   boolean ret = false;

    if ((pPos <= 0) || (pPos > pstlist.size() - 1 ) ) return ret;
    //tokentext directly before div must be ttreturn without space appending it
    TSourceToken lcst = pstlist.get(pPos - 1);
    if (lcst.tokentype != ETokenType.ttreturn) { return ret;}

    //I := Length(lcst.AsText);
    //if not (lcst.AsText[I] = ' ') then result := true;

    if (!(lcst.getAstext().charAt(lcst.getAstext().length()-1) == ' ')) {ret = true;}

    return ret;
}

// place holder function, always return false
boolean isvalidsqlpluscmdInPostgresql(String astr){
    return false;
}

//boolean isvalidsqlpluscmd(String astr){
//    boolean ret = false;
//    if (sqlpluskeywordList.size() == 0){
//    sqlpluskeywordList.put("ACC","1");
//        sqlpluskeywordList.put("ACCEPT","1");
//        sqlpluskeywordList.put("A","1");
//        sqlpluskeywordList.put("APPEND","1");
//        sqlpluskeywordList.put("ATTRIBUTE","1");
//        sqlpluskeywordList.put("BRE","1");
//        sqlpluskeywordList.put("BREAK","1");
//        sqlpluskeywordList.put("BTI","1");
//        sqlpluskeywordList.put("BTITLE","1");
//        sqlpluskeywordList.put("C","1");
//        sqlpluskeywordList.put("CHANGE","1");
//        sqlpluskeywordList.put("CL","1");
//        sqlpluskeywordList.put("CLEAR","1");
//        sqlpluskeywordList.put("COL","1");
//        sqlpluskeywordList.put("COLUMN","1");
//        sqlpluskeywordList.put("COMP","1");
//        sqlpluskeywordList.put("COMPUTE","1");
//        sqlpluskeywordList.put("CONN","1");
//        sqlpluskeywordList.put("CONNECT","1");
//        sqlpluskeywordList.put("COPY","1");
//        sqlpluskeywordList.put("DEF","1");
//        sqlpluskeywordList.put("DEFINE","1");
//        sqlpluskeywordList.put("DEL","1");
//        sqlpluskeywordList.put("DESC","1");
//        sqlpluskeywordList.put("DESCRIBE","1");
//        sqlpluskeywordList.put("DISC","1");
//        sqlpluskeywordList.put("DISCONNECT","1");
//        sqlpluskeywordList.put("ED","1");
//        sqlpluskeywordList.put("EDIT","1");
//        sqlpluskeywordList.put("EXEC","1");
//        sqlpluskeywordList.put("EXIT","1");
//        sqlpluskeywordList.put("QUIT","1");
//        sqlpluskeywordList.put("GET","1");
//        sqlpluskeywordList.put("HELP","1");
//        sqlpluskeywordList.put("HO","1");
//        sqlpluskeywordList.put("HOST","1");
//        sqlpluskeywordList.put("I","1");
//        sqlpluskeywordList.put("INPUT","1");
//        sqlpluskeywordList.put("L","1");
//        sqlpluskeywordList.put("LIST","1");
//        sqlpluskeywordList.put("PASSW","1");
//        sqlpluskeywordList.put("PASSWORD","1");
//        sqlpluskeywordList.put("PAU","1");
//        sqlpluskeywordList.put("PAUSE","1");
//        sqlpluskeywordList.put("PRI","1");
//        sqlpluskeywordList.put("PRINT","1");
//        sqlpluskeywordList.put("PROMPT","1");
//        sqlpluskeywordList.put("REM","1");
//        sqlpluskeywordList.put("REMARK","1");
//        sqlpluskeywordList.put("REPF","1");
//        sqlpluskeywordList.put("REPFOOTER","1");
//        sqlpluskeywordList.put("REPH","1");
//        sqlpluskeywordList.put("REPHEADER","1");
//        sqlpluskeywordList.put("R","1");
//        sqlpluskeywordList.put("RUN","1");
//        sqlpluskeywordList.put("SAV","1");
//        sqlpluskeywordList.put("SAVE","1");
//        sqlpluskeywordList.put("SET","1");
//        sqlpluskeywordList.put("SHO","1");
//        sqlpluskeywordList.put("SHOW","1");
//        sqlpluskeywordList.put("SPO","1");
//        sqlpluskeywordList.put("SPOOL","1");
//        sqlpluskeywordList.put("STA","1");
//        sqlpluskeywordList.put("START","1");
//        sqlpluskeywordList.put("STORE","1");
//        sqlpluskeywordList.put("TIMI","1");
//        sqlpluskeywordList.put("TIMING","1");
//        sqlpluskeywordList.put("TTI","1");
//        sqlpluskeywordList.put("TTITLE","1");
//        sqlpluskeywordList.put("UNDEF","1");
//        sqlpluskeywordList.put("UNDEFINE","1");
//        sqlpluskeywordList.put("VAR","1");
//        sqlpluskeywordList.put("VARIABLE","1");
//        sqlpluskeywordList.put("WHENEVER","1");
//        sqlpluskeywordList.put("@","1");
//        sqlpluskeywordList.put("@@","1");
//    }
//
//    try{
//     ret =   (Integer.parseInt( (String) sqlpluskeywordList.get(astr.toUpperCase()) ) ) == 1 ;
//    }catch(NumberFormatException e){
//        ret = false;
//    }
//
//    return ret;
//}

TSourceToken getprevsolidtoken(TSourceToken ptoken){
    TSourceToken ret = null;
    TSourceTokenList lctokenlist = ptoken.container;
    if (lctokenlist != null){
        if ((ptoken.posinlist > 0)  &&  (lctokenlist.size() > ptoken.posinlist-1))
        {
            if( !(
                    (lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttwhitespace)
                    ||(lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttreturn)
                ||(lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttsimplecomment)
                ||(lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttbracketedcomment)
            ))
            { ret = lctokenlist.get(ptoken.posinlist-1);}
            else
            { ret = lctokenlist.nextsolidtoken(ptoken.posinlist-1,-1,false);}
        }
    }
    return ret;
}

    void doredshifttexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;


        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {


            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                    if (insqlpluscmd){
                        asourcetoken.insqlpluscmd = true;
                    }
                    break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                        isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                            lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.toString().equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                                // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                                && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.toString()) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    if (asourcetoken.tokencode == TBaseType.rrw_redshift_rowtype) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }

                }
            }

            //flexer.yylexwrap(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if (asourcetoken != null){
                yychar = asourcetoken.tokencode;
            }else{
                yychar = 0;

                if (waitingreturnforfloatdiv)
                { // / at the end of line treat as sqlplus command
                    //isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                        lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }
            }

            if ((yychar == 0) && (prevst != null))
            {
            }

        }// while
    }

    void doansitexttotokenlist(){
        dodb2sqltexttotokenlist();
    }
    void dogaussdbtexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;

        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                    if (insqlpluscmd){
                        asourcetoken.insqlpluscmd = true;
                    }
                    break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                        isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                            lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.toString().equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                                && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.toString()) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    // the inner keyword tokentext should be converted to TBaseType.ident when
                    // next solid tokentext is not join

                    if (prevst != null)
                    {
                        if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                        {
                            if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                            {prevst.tokencode = TBaseType.ident;}
                        }


                        if ((prevst.tokencode == TBaseType.rrw_not)
                                && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                        {
                            prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                        }

                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_inner)
                    {
                        prevst = asourcetoken;
                    }
                    else if (asourcetoken.tokencode == TBaseType.rrw_not)
                    {
                        prevst = asourcetoken;
                    }
                    else
                    { prevst = null;}

                    if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                            ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                    {
                        // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                        // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                        // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                        lcprevst = getprevsolidtoken(asourcetoken);
                        if (lcprevst != null)
                        {
                            if (lcprevst.tokencode == TBaseType.rrw_for)
                                lcprevst.tokencode = TBaseType.rw_for1;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                        //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                        TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                        if ( stKeep != null){
                            stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                        }
                    }

                    if ((asourcetoken.tokencode == TBaseType.rrw_postgresql_rowtype)||(asourcetoken.tokencode == TBaseType.rrw_postgresql_type)) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.JSON_EXIST) {
                        TSourceToken stPercent = asourcetoken.searchToken('=', -1);
                        if (stPercent != null) {    // = ?, ? after = should not be treated as json exist operator
                            asourcetoken.tokencode = TBaseType.ident;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_update) { // on conflict do update in insert statement
                        TSourceToken stDo = asourcetoken.searchToken(TBaseType.rrw_do, -1);
                        if (stDo != null) {
                            asourcetoken.tokencode = TBaseType.rrw_postgresql_do_update;
                        }
                    }


                }
            }

            //flexer.yylexwrap(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if (asourcetoken != null){
                yychar = asourcetoken.tokencode;
            }else{
                yychar = 0;

                if (waitingreturnforfloatdiv)
                { // / at the end of line treat as sqlplus command
                    //isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                        lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }

            }

            if ((yychar == 0) && (prevst != null))
            {
                if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
                { prevst.tokencode = TBaseType.ident;}
            }


        }


    }
    void dopostgresqltexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;

        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                    if (insqlpluscmd){
                        asourcetoken.insqlpluscmd = true;
                    }
                    break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                        isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                            lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.toString().equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                                // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                                && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.toString()) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    // the inner keyword tokentext should be convert to TBaseType.ident when
                    // next solid tokentext is not join

                    if (prevst != null)
                    {
                        if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                        {
                            if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                            {prevst.tokencode = TBaseType.ident;}
                        }


                        if ((prevst.tokencode == TBaseType.rrw_not)
                                && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                        {
                            prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                        }

                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_inner)
                    {
                        prevst = asourcetoken;
                    }
                    else if (asourcetoken.tokencode == TBaseType.rrw_not)
                    {
                        prevst = asourcetoken;
                    }
                    else
                    { prevst = null;}

                    if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                            ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                    {
                        // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                        // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                        // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                        lcprevst = getprevsolidtoken(asourcetoken);
                        if (lcprevst != null)
                        {
                            if (lcprevst.tokencode == TBaseType.rrw_for)
                                lcprevst.tokencode = TBaseType.rw_for1;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                        //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                        TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                        if ( stKeep != null){
                            stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                        }
                    }

                    if ((asourcetoken.tokencode == TBaseType.rrw_postgresql_rowtype)||(asourcetoken.tokencode == TBaseType.rrw_postgresql_type)) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.JSON_EXIST) {
                        TSourceToken stPercent = asourcetoken.searchToken('=', -1);
                        if (stPercent != null) {    // = ?, ? after = should not be treated as json exist operator
                            asourcetoken.tokencode = TBaseType.ident;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_update) { // on conflict do update in insert statement
                        TSourceToken stDo = asourcetoken.searchToken(TBaseType.rrw_do, -1);
                        if (stDo != null) {
                            asourcetoken.tokencode = TBaseType.rrw_postgresql_do_update;
                        }
                    }


                }
            }

            //flexer.yylexwrap(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if (asourcetoken != null){
                yychar = asourcetoken.tokencode;
            }else{
                yychar = 0;

                if (waitingreturnforfloatdiv)
                { // / at the end of line treat as sqlplus command
                    //isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                        lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }

            }

            if ((yychar == 0) && (prevst != null))
            {
                if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
                { prevst.tokencode = TBaseType.ident;}
            }


        }


    }

void dosnowflakesqltexttotokenlist(){

    boolean insqlpluscmd = false;
    boolean isvalidplace = true;
    boolean waitingreturnforfloatdiv = false;
    boolean waitingreturnforsemicolon = false;
    boolean continuesqlplusatnewline = false;

    TSourceToken lct = null, prevst = null;

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
      {
        sourcetokenlist.add(asourcetoken);
        switch(yychar){
            case TBaseType.cmtdoublehyphen:
            case TBaseType.cmtslashstar:
            case TBaseType.lexspace:{
               if (insqlpluscmd){
                   asourcetoken.insqlpluscmd = true;
               }
               break;
            }
            case TBaseType.lexnewline:{
                if (insqlpluscmd){
                    insqlpluscmd = false;
                    isvalidplace = true;

                    if (continuesqlplusatnewline) {
                        insqlpluscmd = true;
                        isvalidplace = false;
                        asourcetoken.insqlpluscmd = true ;
                    }
                }

                if (waitingreturnforsemicolon){
                  isvalidplace = true;
                }
                if (waitingreturnforfloatdiv)
                {
                    isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                      lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }
                flexer.insqlpluscmd = insqlpluscmd;
                break;
            } //case newline
            default:{
                //solid tokentext
                continuesqlplusatnewline = false;
                waitingreturnforsemicolon = false;
                waitingreturnforfloatdiv = false;
                if (insqlpluscmd)
                {
                    asourcetoken.insqlpluscmd = true ;
                    if (asourcetoken.getAstext().equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                }
                else
                {
                    if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                    {waitingreturnforsemicolon = true;}
                    if ( (asourcetoken.tokentype == ETokenType.ttslash)
                      // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                       && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                    {
                        lct = asourcetoken;
                        waitingreturnforfloatdiv = true;
                    }
                    if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.toString()) )
                    {
                        asourcetoken.tokencode = TBaseType.sqlpluscmd;
                        if (asourcetoken.tokentype != ETokenType.ttslash)
                        {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                        insqlpluscmd = true;
                        flexer.insqlpluscmd = insqlpluscmd;
                    }
                }
                isvalidplace = false;

                // the inner keyword tokentext should be convert to TBaseType.ident when
                // next solid tokentext is not join

                 if (prevst != null)
                 {
                    if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                    {
                        if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                        {prevst.tokencode = TBaseType.ident;}
                    }


                    if ((prevst.tokencode == TBaseType.rrw_not)
                    && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                    {
                          prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                    }

                 }

                if (asourcetoken.tokencode == TBaseType.rrw_inner)
                {
                    prevst = asourcetoken;
                }
                else if (asourcetoken.tokencode == TBaseType.rrw_not)
                {
                     prevst = asourcetoken;
                }
                else
                { prevst = null;}


            }
        }

         //flexer.yylexwrap(asourcetoken);
         asourcetoken = getanewsourcetoken();
         if (asourcetoken != null){
             yychar = asourcetoken.tokencode;
         }else{
           yychar = 0;

             if (waitingreturnforfloatdiv)
             { // / at the end of line treat as sqlplus command
                 //isvalidplace = true;
                 lct.tokencode = TBaseType.sqlpluscmd;
                 if (lct.tokentype != ETokenType.ttslash){
                   lct.tokentype = ETokenType.ttsqlpluscmd;
                 }
             }

         }

          if ((yychar == 0) && (prevst != null))
          {
              if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
              { prevst.tokencode = TBaseType.ident;}
          }


      } // while


}

    void doverticatexttotokenlist(){
        dopostgresqltexttotokenlist();
    }

    void docouchbasesqltexttotokenlist(){
        dopostgresqltexttotokenlist();
    }

    void doclickhousesqltexttotokenlist(){
        dopostgresqltexttotokenlist();
    }

    /**
     * Turn one token: `schema.table_name` into 3 tokens: `schema` . `table_name`
     * @param asourcetoken
     * @return
     */
    int splitQualifiedNameInBacktick(TSourceToken asourcetoken){
        int yychar = 0;

        List<String> elephantList = Arrays.asList(TBaseType.getTextWithoutQuoted(asourcetoken.toString()).split("\\."));
        int p = 0,offset=0;
        for(String s:elephantList){
            TSourceToken pst = new TSourceToken("`"+s+"`");
            pst.tokencode = asourcetoken.tokencode;
            pst.tokentype =  asourcetoken.tokentype;
            pst.tokenstatus = asourcetoken.tokenstatus;
            pst.lineNo = asourcetoken.lineNo;
            pst.columnNo = asourcetoken.columnNo+offset;
            if (p==0) offset++;// this count the first ` token
            offset = offset+s.length();
            pst.container = sourcetokenlist;
            if (p>0){// 第一个token使用被拆分前那个token的位置，从第二个开始的token，需要先把列表的位置指针加 1
                sourcetokenlist.curpos = sourcetokenlist.curpos+1;
            }
            pst.posinlist = sourcetokenlist.curpos;

            sourcetokenlist.add(pst);
            yychar = pst.tokencode;

            if (p != elephantList.size()-1){
                //`schema.table_name`, add period token in the middle of the backtick included identifier.
                TSourceToken periodst = new TSourceToken(".");
                periodst.tokencode = '.';
                periodst.tokentype =  ETokenType.ttperiod;
                periodst.tokenstatus = asourcetoken.tokenstatus;
                periodst.lineNo = asourcetoken.lineNo;
                periodst.columnNo = asourcetoken.columnNo + offset;
                offset++;
                periodst.container = sourcetokenlist;
                sourcetokenlist.curpos = sourcetokenlist.curpos+1;
                periodst.posinlist = sourcetokenlist.curpos;
                sourcetokenlist.add(periodst);
                yychar = periodst.tokencode;
            }

            p++;
            //System.out.println(s);
        }

        return yychar;

    }

    void dobigquerysqltexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;


        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            if (asourcetoken != null){
                sourcetokenlist.add(asourcetoken);
            }

            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            yychar = asourcetoken.tokencode;

            // `schema.table_name`
            if ((asourcetoken.tokencode == TBaseType.ident)
                    && (asourcetoken.toString().startsWith("`"))&& (asourcetoken.toString().endsWith("`"))
                    && (asourcetoken.toString().indexOf(".")>0)
            ){
                yychar = splitQualifiedNameInBacktick(asourcetoken);
                asourcetoken = null;
            }
        }
    }

    void dosoqlsqltexttotokenlist(){
        domssqlsqltexttotokenlist();
    }

void dogreenplumtexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;

        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
          {
            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                   if (insqlpluscmd){
                       asourcetoken.insqlpluscmd = true;
                   }
                   break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                      isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                          lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.toString().equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                          // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                           && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.toString()) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    // the inner keyword tokentext should be convert to TBaseType.ident when
                    // next solid tokentext is not join

                     if (prevst != null)
                     {
                        if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                        {
                            if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                            {prevst.tokencode = TBaseType.ident;}
                        }


                        if ((prevst.tokencode == TBaseType.rrw_not)
                        && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                        {
                              prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                        }

                     }

                    if (asourcetoken.tokencode == TBaseType.rrw_inner)
                    {
                        prevst = asourcetoken;
                    }
                    else if (asourcetoken.tokencode == TBaseType.rrw_not)
                    {
                         prevst = asourcetoken;
                    }
                    else
                    { prevst = null;}

                    if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                    ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                    {
                        // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                        // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                        // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                        lcprevst = getprevsolidtoken(asourcetoken);
                        if (lcprevst != null)
                        {
                            if (lcprevst.tokencode == TBaseType.rrw_for)
                              lcprevst.tokencode = TBaseType.rw_for1;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                        //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                        TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                        if ( stKeep != null){
                            stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_greenplum_rowtype) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }


                }
            }

             //flexer.yylexwrap(asourcetoken);
             asourcetoken = getanewsourcetoken();
             if (asourcetoken != null){
                 yychar = asourcetoken.tokencode;
             }else{
               yychar = 0;

                 if (waitingreturnforfloatdiv)
                 { // / at the end of line treat as sqlplus command
                     //isvalidplace = true;
                     lct.tokencode = TBaseType.sqlpluscmd;
                     if (lct.tokentype != ETokenType.ttslash){
                       lct.tokentype = ETokenType.ttsqlpluscmd;
                     }
                 }

             }

              if ((yychar == 0) && (prevst != null))
              {
                  if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
                  { prevst.tokencode = TBaseType.ident;}
              }


          }


    }

void donetezzatexttotokenlist(){

    boolean insqlpluscmd = false;
    boolean isvalidplace = true;
    boolean waitingreturnforfloatdiv = false;
    boolean waitingreturnforsemicolon = false;
    boolean continuesqlplusatnewline = false;

    TSourceToken lct = null, prevst = null;

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
      {
        sourcetokenlist.add(asourcetoken);
        switch(yychar){
            case TBaseType.cmtdoublehyphen:
            case TBaseType.cmtslashstar:
            case TBaseType.lexspace:{
               if (insqlpluscmd){
                   asourcetoken.insqlpluscmd = true;
               }
               break;
            }
            case TBaseType.lexnewline:{
                if (insqlpluscmd){
                    insqlpluscmd = false;
                    isvalidplace = true;

                    if (continuesqlplusatnewline) {
                        insqlpluscmd = true;
                        isvalidplace = false;
                        asourcetoken.insqlpluscmd = true ;
                    }
                }

                if (waitingreturnforsemicolon){
                  isvalidplace = true;
                }
                if (waitingreturnforfloatdiv)
                {
                    isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                      lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }
                flexer.insqlpluscmd = insqlpluscmd;
                break;
            } //case newline
            default:{
                //solid tokentext
                continuesqlplusatnewline = false;
                waitingreturnforsemicolon = false;
                waitingreturnforfloatdiv = false;
                if (insqlpluscmd)
                {
                    asourcetoken.insqlpluscmd = true ;
                    if (asourcetoken.getAstext().equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                }
                else
                {
                    if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                    {waitingreturnforsemicolon = true;}
                    if ( (asourcetoken.tokentype == ETokenType.ttslash)
                      // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                       && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                    {
                        lct = asourcetoken;
                        waitingreturnforfloatdiv = true;
                    }
                    if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.toString()) )
                    {
                        asourcetoken.tokencode = TBaseType.sqlpluscmd;
                        if (asourcetoken.tokentype != ETokenType.ttslash)
                        {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                        insqlpluscmd = true;
                        flexer.insqlpluscmd = insqlpluscmd;
                    }
                }
                isvalidplace = false;

                // the inner keyword tokentext should be convert to TBaseType.ident when
                // next solid tokentext is not join

                 if (prevst != null)
                 {
                    if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                    {
                        if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                        {prevst.tokencode = TBaseType.ident;}
                    }


                    if ((prevst.tokencode == TBaseType.rrw_not)
                    && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                    {
                          prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                    }

                 }

                if (asourcetoken.tokencode == TBaseType.rrw_inner)
                {
                    prevst = asourcetoken;
                }
                else if (asourcetoken.tokencode == TBaseType.rrw_not)
                {
                     prevst = asourcetoken;
                }
                else
                { prevst = null;}

                if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                {
                    // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                    // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                    // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        if (lcprevst.tokencode == TBaseType.rrw_for)
                          lcprevst.tokencode = TBaseType.rw_for1;
                    }
                }

                if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                    //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                    TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                    if ( stKeep != null){
                        stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                    }
                }


            }
        }

         //flexer.yylexwrap(asourcetoken);
         asourcetoken = getanewsourcetoken();
         if (asourcetoken != null){
             yychar = asourcetoken.tokencode;
         }else{
           yychar = 0;

             if (waitingreturnforfloatdiv)
             { // / at the end of line treat as sqlplus command
                 //isvalidplace = true;
                 lct.tokencode = TBaseType.sqlpluscmd;
                 if (lct.tokentype != ETokenType.ttslash){
                   lct.tokentype = ETokenType.ttsqlpluscmd;
                 }
             }

         }

          if ((yychar == 0) && (prevst != null))
          {
              if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
              { prevst.tokencode = TBaseType.ident;}
          }


      }
}

private boolean spaceAtTheEndOfReturnToken(String s){
    int pos = 0;
    if (s == null) return false;
    if (s.length() == 0) return false;

    return ((s.charAt(s.length() -1 ) == ' ')||(s.charAt(s.length() -1 ) == '\t'));
}

private int countLines(String s){
    int pos = 0, lf = 0, cr = 0;

    while (pos < s.length()) {
        if (s.charAt(pos) == '\r') {
            cr++;
            pos++;
            continue;
        }
        if (s.charAt(pos) == '\n') {
            lf++;
            pos++;
            continue;
        }

        if (s.charAt(pos) == ' ') {
            pos++;
            continue;
        }
        break;
    }

    if (lf >= cr) return lf;
    else return  cr;

 }


void dodb2sqltexttotokenlist(){
// treat echo, ! of db2 at the begin of each line as
// oracle sqlplus command

    boolean insqlpluscmd = false;
    boolean isvalidplace = true;
    int jdbc_escape_nest = 0;
//  isSolidTokenStart := false;

//  tempsourcetokenlist.clear;

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;


    while (yychar > 0)
    {

        switch (yychar){
          case TBaseType.lexnewline:
          {
              insqlpluscmd = false;
              isvalidplace = true;
              break;
          }
          case TBaseType.cmtdoublehyphen:
          {
              if (asourcetoken.toString().toLowerCase().indexOf("scriptoptions")>=0)
              {
                  asourcetoken.tokencode = TBaseType.scriptoptions;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
              }

              if (insqlpluscmd)
              {
                  asourcetoken.insqlpluscmd = true;
              }
            break;
          }
          case TBaseType.rrw_jdbc_escape_fn: {
              jdbc_escape_nest++;
              asourcetoken.tokencode = TBaseType.lexspace;
            break;
          }
          case TBaseType.rrw_jdbc_escape_end: {
                jdbc_escape_nest--;
                asourcetoken.tokencode = TBaseType.lexspace;
                break;
          }
          default: //solid tokentext
          {
              if ((asourcetoken.tokencode == TBaseType.rrw_rr)
                    || (asourcetoken.tokencode ==  TBaseType.rrw_rs)
                    || (asourcetoken.tokencode ==  TBaseType.rrw_cs)
                    || (asourcetoken.tokencode ==  TBaseType.rrw_ur)
                  )
              {
                 // change with keyword in isolation clause to rrw_with_isolation
                  // so opt_restriction_clause in create view worked correctly
                  lcprevst = getprevsolidtoken(asourcetoken);
                  if (lcprevst != null)
                  {
                      if (lcprevst.tokencode == TBaseType.rrw_with)
                        lcprevst.tokencode = TBaseType.rrw_with_isolation;
                  }
              }

              if (insqlpluscmd)
              {
                  //asourcetoken.TokenCode := TBaseType.sqlpluscmd;
                  asourcetoken.insqlpluscmd = true;
              }
              else
              {
                  if (isvalidplace){
                   // System.out.println(asourcetoken.astext);
                  if (TBaseType.mycomparetext(asourcetoken.toString().toLowerCase(),"echo") == 0 )
                  {
                     // System.out.println("finded ");
                      asourcetoken.tokencode = TBaseType.sqlpluscmd;
                      if (asourcetoken.tokentype != ETokenType.ttslash)
                      {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                      asourcetoken.insqlpluscmd = true;
                      insqlpluscmd = true;
                  }
                }
              }
              isvalidplace = false;

              break;
          } //end of solid tokentext
        } //switch

//  TBaseType.lexnewline

        sourcetokenlist.add(asourcetoken);

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;

    } //while

}

    void doteradatatexttotokenlist(){

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;

            if ((asourcetoken.tokencode == TBaseType.rrw_casespecific)
                ||(asourcetoken.tokencode == TBaseType.rrw_teradata_cs))
            {
                    /* change not to not1 to make "not null column constriants" work correctly */
                //      |RW_NOT1 RW_CASESPECIFIC

                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == TBaseType.rrw_not)
                      lcprevst.tokencode = TBaseType.rw_not1;
                }
            } else if (asourcetoken.tokencode == ';'){
                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == ';')
                        asourcetoken.tokencode = TBaseType.sqlpluscmd;
                }
            } else if (asourcetoken.tokencode == TBaseType.variable){ // mantisbt/view.php?id=972
                if (asourcetoken.toString().toLowerCase().endsWith("begin")){ //MYPROC :BEGIN
                    asourcetoken.tokencode = TBaseType.rrw_begin;
                    asourcetoken.tokentype = ETokenType.ttkeyword;

                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        lcprevst.tokencode = TBaseType.mslabel;
                    }

                }
            }else if (asourcetoken.tokencode == TCustomLexer.UNICODE_ENCODE_ID){
                if (asourcetoken.toString().endsWith("008D")){ // REVERSE LINE FEED, https://codepoints.net/U+008D?lang=en
                    asourcetoken.tokencode = TBaseType.lexspace;
                }else{
                    asourcetoken.tokencode = TBaseType.ident;
                }
            }


            yychar = asourcetoken.tokencode;
        }

    }

void  docommonsqltexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}
    void  doodbcsqltexttotokenlist(){
        boolean insideODBC = false;
        TSourceToken odbcPrefix = null;
        domssqlsqltexttotokenlist();
        for (int i=0 ; i<sourcetokenlist.size();i++) {
            TSourceToken ast = sourcetokenlist.get(i);
            if ((ast.tokencode == '{')||(ast.tokencode == TBaseType.odbc_esc_prefix))
            {
                insideODBC = true;
                odbcPrefix = ast;
            }
            if ((ast.tokencode == '}')||(ast.tokencode == TBaseType.odbc_esc_terminator))
            {
                insideODBC = false;
                odbcPrefix = null;
            }
            if (((ast.tokencode == TBaseType.rrw_odbc_d)
                    ||(ast.tokencode == TBaseType.rrw_odbc_t)
                    ||(ast.tokencode == TBaseType.rrw_odbc_ts)
                    ||(ast.tokencode == TBaseType.rrw_odbc_fn)
                    ||(ast.tokencode == TBaseType.rrw_odbc_oj)
                )&&(!insideODBC)) {
                ast.tokencode = TBaseType.ident;
            }

            if ((ast.tokencode == TBaseType.rrw_call)&&(insideODBC)){
                odbcPrefix.setLinkToken(ast);
            }
        }
    }

void  dodaxsqltexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}


void  dohanasqltexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}

void  dohivetexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        if (asourcetoken != null){
            sourcetokenlist.add(asourcetoken);
        }
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        if (asourcetoken.tokencode == TBaseType.rrw_map){
            TSourceToken token = asourcetoken.searchToken(')',-1);
            if (token != null){
                asourcetoken.tokencode = TBaseType.ident;
            }
        }else if (asourcetoken.tokencode == '('){
//            TSourceToken token = asourcetoken.searchToken(TBaseType.ident,-1);
//            if (token != null){
//                token.tokencode = TBaseType.HIVE_FUNC_IDENT;
//            }
        }
        yychar = asourcetoken.tokencode;

        // `schema.table_name`
        if ((asourcetoken.tokencode == TBaseType.ident)
                && (asourcetoken.toString().startsWith("`"))&& (asourcetoken.toString().endsWith("`"))
                && (asourcetoken.toString().indexOf(".")>0)
        ){
            yychar = splitQualifiedNameInBacktick(asourcetoken);
            asourcetoken = null;
        }

    }

}

void  doimpalatexttotokenlist(){
    dohivetexttotokenlist();
}

void checkMySQLCommentToken(TSourceToken cmtToken){
//    if (cmtToken.tokencode == TBaseType.cmtdoublehyphen){
//        if ((cmtToken.toString().startsWith("--"))){
//            if (cmtToken.toString().length() > 2){
//                if (cmtToken.toString().charAt(2) != ' '){
//                   this.syntaxErrors.add(new TSyntaxError(cmtToken.toString(),
//                           cmtToken.lineNo,
//                           cmtToken.columnNo,
//                           "requires a space after the initial \"--\" ",EErrorType.sperror,10010
//                           ));
//                }
//            }
//            // there should be
//        }
//    }
}
    void dosparksqltexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size() - 1).getAstext();
            }


            if (asourcetoken.tokencode == TBaseType.rrw_rollup)
            {
                //      with rollup

                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == TBaseType.rrw_with)
                        lcprevst.tokencode = TBaseType.with_rollup;
                }
            }

            yychar = asourcetoken.tokencode;
        }

    }

    void doathenatexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size() - 1).getAstext();
            }


            yychar = asourcetoken.tokencode;
        }

    }

    void dodatabrickstexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size() - 1).getAstext();
            }


            yychar = asourcetoken.tokencode;
        }

    }
    void doprestotexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size() - 1).getAstext();
            }


            yychar = asourcetoken.tokencode;
        }

    }

void  domysqltexttotokenlist(){

        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;
        checkMySQLCommentToken(asourcetoken);

        if ((asourcetoken.tokencode == TBaseType.rrw_mysql_delimiter)){
            startDelimiter = true;
        }

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size() - 1).getAstext();
            }

            if ((asourcetoken.tokencode == TBaseType.rrw_mysql_delimiter)){
                startDelimiter = true;
            }

            if (asourcetoken.tokencode == TBaseType.rrw_rollup)
            {
                //      with rollup

                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == TBaseType.rrw_with)
                      lcprevst.tokencode = TBaseType.with_rollup;
                }
            }

            if( (asourcetoken.tokencode == TBaseType.rrw_mysql_d)
                    ||(asourcetoken.tokencode == TBaseType.rrw_mysql_t)
                    ||(asourcetoken.tokencode == TBaseType.rrw_mysql_ts)
                    )
            {
                //      odbc date constant { d 'str' }
                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode != '{')
                        asourcetoken.tokencode = TBaseType.ident;
                }
            }


            yychar = asourcetoken.tokencode;
        }

    }

void  doMdxtexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}

void dosqltexttotokenlist(){
    switch(dbVendor){
        case dbvmssql:
        case dbvazuresql:
         {
            domssqlsqltexttotokenlist();
            break;
        }
        case dbvinformix:{
            doinformixtexttotokenlist();
            break;
        }
        case dbvsybase:{
            dosybasesqltexttotokenlist();
            break;
        }
        case dbvdb2:{
            dodb2sqltexttotokenlist();
            break;
        }
        case dbvteradata:{
            doteradatatexttotokenlist();
            break;
        }
        case dbvpostgresql:{
            dopostgresqltexttotokenlist();
            break;
        }
        case dbvredshift:{
            doredshifttexttotokenlist();
            break;
        }
        case dbvgreenplum:{
            dogreenplumtexttotokenlist();
            break;
        }
        case dbvmdx: {
            doMdxtexttotokenlist();
            break;
        }
        case dbvnetezza:{
            donetezzatexttotokenlist();
            break;
        }
        case dbvhive:{
            dohivetexttotokenlist();
            break;
        }
        case dbvimpala:{
            doimpalatexttotokenlist();
            break;
        }
        case dbvmysql:{
            domysqltexttotokenlist();
            break;
        }
        case dbvhana:{
            dohanasqltexttotokenlist();
            break;
        }
        case dbvdax:{
            dodaxsqltexttotokenlist();
            break;
        }
        case dbvodbc:{
            doodbcsqltexttotokenlist();
            break;
        }
        case dbvvertica:{
            doverticatexttotokenlist();
            break;
        }
        case dbvopenedge:{
            domssqlsqltexttotokenlist();
            break;
        }
        case dbvcouchbase:{
            docouchbasesqltexttotokenlist();
            break;
        }
        case dbvsnowflake:{
            dosnowflakesqltexttotokenlist();
            break;
        }
        case dbvbigquery:{
            dobigquerysqltexttotokenlist();
            break;
        }
        case dbvclickhouse:{
            doclickhousesqltexttotokenlist();
            break;
        }
        case dbvsoql:{
            dosoqlsqltexttotokenlist();
            break;
        }
        case dbvsparksql:{
            dosparksqltexttotokenlist();
            break;
        }
        case dbvpresto:{
            doprestotexttotokenlist();
            break;
        }
        case dbvathena:{
            doathenatexttotokenlist();
            break;
        }
        case dbvdatabricks:{
            dodatabrickstexttotokenlist();
            break;
        }
        case dbvgaussdb:{
            dogaussdbtexttotokenlist();
            break;
        }
        case dbvansi:{
            doansitexttotokenlist();
            break;
        }
        default:{
            docommonsqltexttotokenlist();            
        }
    }

    doAfterTokenize();

    TBaseType.resetTokenChain(sourcetokenlist,0);

    processTokensInTokenTable(dbVendor);
    processTokensBeforeParse(dbVendor);

    closeFileStream();

    if (tokenListHandle != null){
        tokenListHandle.processTokenList(sourcetokenlist);
    }
}

void doAfterTokenize(){

    int leftParenCount = 0;
    int rightParenCount = 0;
    int leftIndex = 0;
    int rightIndex = sourcetokenlist.size() - 1;

    // Count opening parentheses at the beginning
    while (leftIndex < sourcetokenlist.size() && sourcetokenlist.get(leftIndex).tokencode == '(') {
        leftParenCount++;
        leftIndex++;
    }

    // Count closing parentheses at the end
    while (rightIndex >= 0 && sourcetokenlist.get(rightIndex).tokencode == ')') {
        rightParenCount++;
        rightIndex--;
    }

    // Set matching parentheses to be ignored
    int parensToIgnore = Math.min(leftParenCount, rightParenCount);
    // if there is a semicolon before the right parenthesis, set the semicolon to be ignored
    // mantisbt/view.php?id=3690

    if ((parensToIgnore > 0) && (sourcetokenlist.get(sourcetokenlist.size() - 1 - (parensToIgnore - 1) - 1).tokencode == ';')){
        // set to whitespace that this semicolon will be ignored during getting raw sql
        sourcetokenlist.get(sourcetokenlist.size() - 1 - (parensToIgnore - 1) - 1).tokentype = ETokenType.ttwhitespace;
        // set to ignore by yacc that this semicolon will be ignored during parsing
        sourcetokenlist.get(sourcetokenlist.size() - 1 - (parensToIgnore - 1) - 1).tokenstatus = ETokenStatus.tsignorebyyacc;
    }
//    for (int i = 0; i < parensToIgnore; i++) {
//        //sourcetokenlist.get(i).tokenstatus = ETokenStatus.tsignorebyyacc;
//        sourcetokenlist.get(i).tokencode = TBaseType.lexspace;
//
//       // sourcetokenlist.get(sourcetokenlist.size() - 1 - i).tokenstatus = ETokenStatus.tsignorebyyacc;
//        sourcetokenlist.get(sourcetokenlist.size() - 1 - i).tokencode = TBaseType.lexspace;
//    }

}



void processTokensBeforeParse(EDbVendor dbVendor){

        // 为确保性能，只在snowflake数据库中处理，因为目前只有snowflake数据库中有连续的分号有用户提出这个需求，其他数据库中暂时不处理
   if (dbVendor != EDbVendor.dbvsnowflake) return;

        // mantisbt/view.php?id=3579
        // if there are consecutive semicolon tokens, mark the second semi colon token as deleted token
    for(int i=0;i<sourcetokenlist.size();i++){
        TSourceToken st = sourcetokenlist.get(i);
        if (st.tokencode == ';'){
            TSourceToken nextToken = st.nextSolidToken();
            if (nextToken != null){
                if (nextToken.tokencode == ';'){
                    nextToken.tokenstatus = ETokenStatus.tsdeleted;
                }
            }
        }
    }

}

void processTokensInTokenTable(EDbVendor dbVendor){
    // 获得所有token后，根据需要对token code进行预处理， token table是 TBaseType.TOKEN_TABLE
     long[][] TOKEN_TABLE1 = flexer.TOKEN_TABLE;

    switch (dbVendor){
        case dbvbigquery:
        case dbvsnowflake:
            // case 1, DO 关键字如果没有发现对于的 FOR, WHILE 等关键字，把 DO 关键字的token code设置为 TBaseType.ident

            if (TOKEN_TABLE1[TBaseType.rrw_do][0] > 0){
                if ((TOKEN_TABLE1[TBaseType.rrw_while][0] == 0)&&(TOKEN_TABLE1[TBaseType.rrw_for][0] == 0)){
                    for(int i=0;i<sourcetokenlist.size();i++){
                        TSourceToken st = sourcetokenlist.get(i);
                        if (st.tokencode == TBaseType.rrw_do){
                            st.tokencode = TBaseType.ident;
                        }
                    }
                }
            }

            break;
    }

}

boolean isDollarFunctionDelimiter(int tokencode, EDbVendor dbVendor){
        return ((tokencode == TBaseType.rrw_postgresql_function_delimiter)&&(dbVendor == EDbVendor.dbvpostgresql))
                ||((tokencode == TBaseType.rrw_greenplum_function_delimiter)&&(dbVendor == EDbVendor.dbvgreenplum))
                ||((tokencode == TBaseType.rrw_redshift_function_delimiter)&&(dbVendor == EDbVendor.dbvredshift))
                ||((tokencode == TBaseType.rrw_snowflake_function_delimiter)&&(dbVendor == EDbVendor.dbvsnowflake))
                ||((tokencode == TBaseType.rrw_clickhouse_function_delimiter)&&(dbVendor == EDbVendor.dbvclickhouse));
}

/**
 * Returns the 1-based line number of the end of the last SQL statement that
 * was successfully recognized during raw-statement separation (e.g. via
 * vendor-specific {@code do*getrawsqlstatements} routines).
 * <p>
 * The value corresponds to the ending line of the last validated statement in
 * the most recent parse operation. Any trailing, incomplete statement at the
 * end of the input is intentionally excluded and will not affect this value.
 * <p>
 * Notes:
 * <ul>
 * <li>The line number is relative to the current input provided to the
 * parser (not an absolute position in an external, larger file).</li>
 * <li>This is useful when splitting huge SQL files by safe statement
 * boundaries — callers can cut the source at this line without risking a
 * partial statement.</li>
 * </ul>
 *
 * @return the 1-based line number of the last validated statement's ending
 *         line, or {@code -1} if no statement has been validated yet
 */
public int getLastLineNoOfLastStatementBeenValidated(){
        if (lastTokenOfStatementBeenValidated != null){
            return (int)lastTokenOfStatementBeenValidated.lineNo;
        }
        return -1;
}

private TSourceToken lastTokenOfStatementBeenValidated;


void doongetrawsqlstatementevent(TCustomSqlStatement pcsqlstatement){
        doongetrawsqlstatementevent(pcsqlstatement,false);
}

void doongetrawsqlstatementevent(TCustomSqlStatement pcsqlstatement, boolean isLastSQL){
    pcsqlstatement.setGsqlparser(this);
    pcsqlstatement.parser = this.fparser;
    pcsqlstatement.plsqlparser = this.fplsqlparser;
    pcsqlstatement.setStartToken(pcsqlstatement.sourcetokenlist.get(0));
    pcsqlstatement.setEndToken(pcsqlstatement.sourcetokenlist.get(pcsqlstatement.sourcetokenlist.size()-1));
    sqlstatements.add(pcsqlstatement);

    if (!isLastSQL){ // 最后一个语句没有经过验证，只是在结束的时候强行加入的，很有可能是语法不正确的，因此我们这里只记录非最后的语句
        lastTokenOfStatementBeenValidated = pcsqlstatement.getEndToken();
    }

    // if stored procedure body is not written in sql or plsql, then, set the token in body to
    if (    ((this.dbVendor == EDbVendor.dbvpostgresql)||(this.dbVendor == EDbVendor.dbvgreenplum) ||(this.dbVendor == EDbVendor.dbvredshift) ||(this.dbVendor == EDbVendor.dbvsnowflake) ||(this.dbVendor == EDbVendor.dbvclickhouse) )
             && (pcsqlstatement instanceof TRoutine)
       ){
         if (!((TRoutine)pcsqlstatement).isBodyInSQL()){
             TSourceToken st;
             boolean inBody = false;
             String routineBodyStr = "";
             for(int i=0;i<pcsqlstatement.sourcetokenlist.size();i++){
                 st = pcsqlstatement.sourcetokenlist.get(i);
                 if (isDollarFunctionDelimiter(st.tokencode,this.dbVendor)
//                         ((st.tokencode == TBaseType.rrw_postgresql_function_delimiter)&&(this.dbVendor == EDbVendor.dbvpostgresql))
//                        ||((st.tokencode == TBaseType.rrw_greenplum_function_delimiter)&&(this.dbVendor == EDbVendor.dbvgreenplum))
//                         ||((st.tokencode == TBaseType.rrw_redshift_function_delimiter)&&(this.dbVendor == EDbVendor.dbvredshift))
//                         ||((st.tokencode == TBaseType.rrw_snowflake_function_delimiter)&&(this.dbVendor == EDbVendor.dbvsnowflake))
                 ){
                     if (!inBody){
                         inBody = true;
                         routineBodyStr = st.toString();
                     }else{
                         inBody = false;
                         routineBodyStr += st.toString();
                         break;
                     }
                     continue;
                 }

                 if (inBody){
                     st.tokencode = TBaseType.sqlpluscmd;
                     routineBodyStr += st.toString();
                 }
             }

             ((TRoutine)pcsqlstatement).setRoutineBody(routineBodyStr);
         }
    }
}

    int doclickhousegetrawsqlstatements(){
        return dopostgresqlgetrawsqlstatements();
    }

    boolean checkTokenPairWithEnd(int tokencode){
        return    ((tokencode == TBaseType.rrw_if)||(tokencode == TBaseType.rrw_case)
                ||(tokencode == TBaseType.rrw_loop)||(tokencode == TBaseType.rrw_repeat)
                ||(tokencode == TBaseType.rrw_while)||(tokencode == TBaseType.rrw_for)
                ||(tokencode == TBaseType.rrw_case)
        );
    }

    int dopostgresqlgetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false, enterDeclare = false;



        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        if (isSinglePLBlock){
            gcurrentsqlstatement = new TCommonBlock(EDbVendor.dbvpostgresql);
        }

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;
            if (isSinglePLBlock){
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }

            if (ast.tokencode == TBaseType.JSON_EXIST){
                TSourceToken stConstant  = ast.searchToken(TBaseType.sconst,1);
                if (stConstant == null){
                    ast.tokencode = TBaseType.ident;
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_POSITION) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '(') {
                        ast.tokencode = TBaseType.rrw_postgresql_POSITION_FUNCTION;
                    }
                }
            }else  if (ast.tokencode == TBaseType.rrw_postgresql_ordinality) {
                TSourceToken lcprevst = getprevsolidtoken(ast);

                if (lcprevst != null) {
                    if (lcprevst.tokencode == TBaseType.rrw_with) {
                        TSourceToken lcnextst = ast.nextSolidToken();
                        if ((lcnextst != null)&&(lcnextst.tokencode == TBaseType.rrw_as)){
                            //  with ordinality as (select 1 as x) select * from ordinality;
                            // don't change with to rrw_postgresql_with_lookahead
                        }else{
                            lcprevst.tokencode = TBaseType.rrw_postgresql_with_lookahead;
                        }

                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_filter) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '(') {

                    }else{
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_jsonb) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '?') {
                        st1.tokencode = TBaseType.OP_JSONB_QUESTION;
                    }
                }
            }
            else if (ast.tokencode == '?') {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == TBaseType.sconst) {
                        ast.tokencode = TBaseType.OP_JSONB_QUESTION;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_values) {
                TSourceToken stParen =  ast.searchToken('(',1);
                if (stParen != null){
                    TSourceToken stInsert  = ast.searchToken(TBaseType.rrw_insert,-ast.posinlist);
                    if (stInsert != null){
                        TSourceToken stSemiColon  = ast.searchToken(';',-ast.posinlist);
                        if ((stSemiColon != null)&&(stSemiColon.posinlist > stInsert.posinlist)){
//                            INSERT INTO test values (16,1), (8,2), (4,4), (2,0), (97, 16);
//                            VALUES (1);
                            // don't treat values(1) as insert values

                        }else{
                            TSourceToken stFrom  = ast.searchToken(TBaseType.rrw_from,-ast.posinlist);
                            if ((stFrom != null)&&(stFrom.posinlist > stInsert.posinlist)){
                                // don't treat values after from keyword as an insert values

                                // insert into inserttest values(10, 20, '40'), (-1, 2, DEFAULT),  ((select 2), (select i from (values(3) ) as foo (i)), 'values are fun!');

                            }else{
                                ast.tokencode = TBaseType.rrw_postgresql_insert_values;
                            }

                        }

                    }
                }
            }

            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd )
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        enterDeclare = false;
                        if (gcurrentsqlstatement.ispgplsql())
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);
                            foundEnd = false;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                                    )
                            {
                                waitingEnd = 1;
                            }
                            else if (ast.tokencode == TBaseType.rrw_declare){
                                enterDeclare = true;
                            }
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.getAstext(),ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsqlplus:{
                    if  (ast.insqlpluscmd)
                    {gcurrentsqlstatement.addtokentolist(ast);}
                    else
                    {
                        gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                    break;
                }//case stsqlplus

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if (ast.tokencode == TBaseType.rrw_postgresql_function_delimiter){
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedurePgStartBody;

                        continue;
                    }

                    if (ast.tokencode == TBaseType.rrw_postgresql_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                            }
                        }
                    }

                    if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (!enterDeclare))
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                            )
                    {
                        waitingEnd++;
                        enterDeclare = false;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_declare)
                    ){
                        enterDeclare = true;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0)
                                && (foundEnd)
                        //        && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())
                        ){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.getAstext());
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    if (( gst == EFindSqlStateType.ststoredprocedure ) && (ast.tokencode  == TBaseType.cmtdoublehyphen)){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    break;
                } //ststoredprocedure

                case ststoredprocedurePgStartBody:{
                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_postgresql_function_delimiter){
                        if (gcurrentsqlstatement.sqlstatementtype == sstDoExecuteBlock){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;

                        }else{
                            gst = EFindSqlStateType.ststoredprocedurePgEndBody;
                            continue;
                        }
                    }

                    break;
                }

                case ststoredprocedurePgEndBody:{

                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    else if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_postgresql_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                            }
                        }
                    }

                    break;
                }
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql)
                        || (gst == EFindSqlStateType.ststoredprocedure)
                        || (gst == EFindSqlStateType.ststoredprocedurePgEndBody)
                        ||(gst == EFindSqlStateType.sterror)||(isSinglePLBlock)
                ))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement,true);
        }

        return syntaxErrors.size();
    }

    /**
     * Functionality
     * Target Identification:
     * Scans for CREATE PROCEDURE and CREATE FUNCTION statements
     * Verifies the language is SQL (for procedures)
     * Identifies dollar-quoted strings ($$...$$) and single-quoted strings after AS keywords
     * Token Expansion:
     * For each identified string literal:
     * Extracts the content between the quotes
     * Tokenizes the extracted SQL code
     * Verifies it's a valid code block (starts with DECLARE or BEGIN)
     * Replaces the original quoted string with expanded tokens in the source token list
     * Special Transformations:
     * Converts BEGIN; to BEGIN TRANSACTION;
     * Handles BEGIN WORK; similarly
     * Special handling for semicolons after END statements
     * Token Chain Management:
     * Preserves original quotes as space tokens
     * Maintains proper token positioning and chaining
     * Resets the token chain to ensure correct output during toString() operations
     * Implementation Details
     * Processes statements in reverse order to maintain correct token positions
     * Uses TBaseType.getStringInsideLiteral() to extract content between quotes
     * Marks original dollar-quoted tokens as deleted (using tsdeleted)
     * Handles multi-line SQL blocks within stored procedure/function bodies
     */
    void expandDollarString(){
        // snowflake, expand dollar string into token list
        TSourceToken st;
        TCustomSqlStatement sql;
        ArrayList<TSourceToken> dollarTokens = new ArrayList<>();
        boolean isSQLLanguage = true;

        // Iterate all create procedure and create function, other sql staetment just skipped
        for(int i=0;i<sqlstatements.size();i++){
            sql = sqlstatements.get(i);
            if (!((sql.sqlstatementtype == sstcreateprocedure)||(sql.sqlstatementtype == sstcreatefunction))) continue;
            isSQLLanguage = true;
            for(int j=0;j<sql.sourcetokenlist.size();j++){
                st = sql.sourcetokenlist.get(j);

                if (sql.sqlstatementtype == sstcreateprocedure){
                    if (st.tokencode == TBaseType.rrw_snowflake_language){
                        TSourceToken lang = st.nextSolidToken();
                        if ((lang != null)&&(!lang.toString().equalsIgnoreCase("sql"))){
                            isSQLLanguage = false;
                        }
                    }
                }

                if (!isSQLLanguage) break;

                if (st.tokencode == TBaseType.sconst){
                    if (st.toString().startsWith("$$")){
                        dollarTokens.add(st); // TODO: 可能会有多个 $$ token? 这种情况再哪个场景下会出现？需要找到对应的测试用例和SQL脚本。否则，uncomment next line
                        // break; // only one $$ token per sql, if we found one, then just break this sql and continue for next sql
                    }else if (st.toString().startsWith("'")){
                        // https://docs.snowflake.com/en/sql-reference/sql/create-procedure
                        // string literal delimiter can be $ or '
                        if (st.prevSolidToken().tokencode == TBaseType.rrw_as){
                            dollarTokens.add(st); // TODO: 可能会有多个 $$ token? 这种情况再哪个场景下会出现？需要找到对应的测试用例和SQL脚本。否则，uncomment next line
                            // break; // only one single quote token per sql, if we found one, then just break this sql and continue for next sql
                        }
                    }
                }
            }//check tokens

            for(int m= dollarTokens.size() - 1; m>=0;m--){

                // Token Expansion:
                // For each identified string literal:
                // Extracts the content between the quotes
                // Tokenizes the extracted SQL code
                // Verifies it's a valid code block (starts with DECLARE or BEGIN)
                // Replaces the original quoted string with expanded tokens in the source token list

                st = dollarTokens.get(m);

                TGSqlParser parser = new TGSqlParser(this.dbVendor);
                //parser.sqltext = TBaseType.stringBlock((int)st.lineNo - 1,(int)st.columnNo)+ TBaseType.getStringInsideLiteral(st.toString());
                // 如果在前面补充空行，会导致 toString() 重新拼接时多出空行；
                // 同时，因为不需要进行解析，所以不需要在前面补充空行以获得准确的错误定位? 这个说法是错误的，因为不加需要的空行，会导致定位不准确 #TODO
                parser.sqltext = TBaseType.getStringInsideLiteral(st.toString()); 
                TSourceToken startQuote = new TSourceToken(st.toString().substring(0,1));
                startQuote.tokencode = TBaseType.lexspace; // 设置为 space，可以在解析时忽略该token，同时在 toString() 时会被原样输出
                TSourceToken endQuote = new TSourceToken(st.toString().substring(0,1));
                endQuote.tokencode = TBaseType.lexspace;

                // use getrawsqlstatements() instead of tokenizeSqltext() to get the source token list because
                // some token will be transformed to other token, which will be processed in dosnowflakegetrawsqlstatements()
                parser.getrawsqlstatements();
                //parser.tokenizeSqltext();

                TSourceToken st2;
                boolean isValidBlock = false;
                for(int k=0;k<parser.sourcetokenlist.size();k++){
                    st2 = parser.sourcetokenlist.get(k);
                    if (st2.isnonsolidtoken()) continue;
                    if ((st2.tokencode == TBaseType.rrw_declare)||(st2.tokencode == TBaseType.rrw_begin)){
                        isValidBlock = true;
                    }else{
                        //
                    }
                    break;
                    //System.out.println(st2.lineNo+","+st2.columnNo+":"+st2.toString());
                }

                if (isValidBlock){
                    TSourceToken semiColon = null;
                    st.tokenstatus = ETokenStatus.tsdeleted;
                    int startPosOfThisSQl = sql.getStartToken().posinlist;

                    sql.sourcetokenlist.add((st.posinlist++) - startPosOfThisSQl,startQuote); // 补上开始引号
                    for(int k=0;k<parser.sourcetokenlist.size();k++) {
                        st2 = parser.sourcetokenlist.get(k);
                        if (st2.tokencode == ';'){
                            semiColon = st2;
                            TSourceToken prevSolidToken = st2.prevSolidToken();
                            if ((prevSolidToken != null) && (prevSolidToken.tokencode == TBaseType.rrw_begin)){
                                // begin;  => begin transaction;
                                prevSolidToken.tokencode = TBaseType.rrw_snowflake_begin_transaction;
                            }
                        }
                        if ((st2.tokencode == TBaseType.rrw_snowflake_work)||(st2.tokencode == TBaseType.rrw_snowflake_transaction)){
                            // begin work;  => begin transaction;
                            TSourceToken prevSolidToken = st2.prevSolidToken();
                            if ((prevSolidToken != null) && (prevSolidToken.tokencode == TBaseType.rrw_begin)){
                                // begin;  => begin transaction;
                                prevSolidToken.tokencode = TBaseType.rrw_snowflake_begin_transaction;
                            }
                        }
                        sql.sourcetokenlist.add( (st.posinlist++) - startPosOfThisSQl,st2);
                        //st.posinlist++;
                    }
                    if (semiColon != null){
                        if (semiColon.prevSolidToken().tokencode == TBaseType.rrw_end){
                            // 设置为 space，可以在解析时忽略该token，同时在 toString() 时会被原样输出
                            // semiColon.tokenstatus = ETokenStatus.tsdeleted;
                            semiColon.tokencode = TBaseType.lexspace;
                        }
                    }

                    sql.sourcetokenlist.add( (st.posinlist++) - startPosOfThisSQl,endQuote); // 补上结束引号
                    TBaseType.resetTokenChain(sql.sourcetokenlist,0); // 重新设置token链,确保新插入的token在 toString() 时能被访问到，从而确保 toString() 输出正确结果
                }
            }

            dollarTokens.clear();
        }//statement

    }

    int domssqlgetrawsqlstatements(){

        // Original implementation follows...
        int errorcount = 0;
        int case_end_nest = 0;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        int lcblocklevel = 0;
        int lctrycatchlevel = 0;
        TSourceToken lcprevsolidtoken = null,lcnextsolidtoken,lcnnextsolidtoken;
        TSourceToken ast = null;
        int i,lcMergeInSelectNested = 0;
        boolean lcisendconversation, lcstillinsql,lcMergeInSelect = false;

        for (i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (ast.tokencode == TBaseType.rrw_for){
                TSourceToken st1 = ast.nextSolidToken();
                if ((st1 != null) && (st1.tokencode == TBaseType.rrw_system_time)){
                    ast.tokencode = TBaseType.rw_for_system_time;
                }
            }

            if (lcMergeInSelect){
                if (ast.tokencode == '(') lcMergeInSelectNested++;
                if (ast.tokencode == ')'){
                    lcMergeInSelectNested--;
                    if (lcMergeInSelectNested == 0){
                        lcMergeInSelect = false;
                    }
                }
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }
            if ( ast.tokenstatus == ETokenStatus.tsignoredbygetrawstatement )
            {
                //tsignoredbygetrawstatement is set when cte is found in dbcmds.findcte function
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }

            if (ast.tokencode == TBaseType.rrw_minus){
                TSourceToken st1 = ast.searchToken('(',1);
                if (st1 == null){
                    st1 = ast.searchToken(TBaseType.rrw_select,1);
                    if (st1 == null){
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_merge){
                TSourceToken st1 = ast.nextSolidToken();
                if (st1.tokencode == TBaseType.rrw_join){
                    ast.tokencode = TBaseType.rrw_merge2_sqlserver;
                }
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '(')){
                    lcMergeInSelect = true;
                    lcMergeInSelectNested++;
                    gcurrentsqlstatement.addtokentolist(ast);
                    continue;
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_value){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_value;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_modify){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_modify;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_query){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_query;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_exist){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_exist;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_nodes){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_nodes;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_check){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == TBaseType.rrw_with)){
                    lcprevsolidtoken.tokencode = TBaseType.rrw_sqlserver_check_with;
                }
//            }else if (ast.tokencode == TBaseType.rrw_for){
//                TSourceToken st1 = ast.nextSolidToken();
//                if ((st1 != null) && (st1.tokencode == TBaseType.rrw_system_time)){
//                    ast.tokencode = TBaseType.rw_for_system_time;
//                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_next){
                TSourceToken st1 = ast.nextSolidToken();
                if ((st1 != null) && (st1.tokencode == '.')){
                    ast.tokencode = TBaseType.ident;
                }
            }else if (ast.tokencode == TBaseType.rrw_fetch){
                if ((lcprevsolidtoken != null)&&((lcprevsolidtoken.tokencode == TBaseType.rrw_sqlserver_row)||(lcprevsolidtoken.tokencode == TBaseType.rrw_sqlserver_rows))){
                      TSourceToken prev2 = lcprevsolidtoken.searchToken(TBaseType.rrw_open,-1);
                      if (prev2 == null){
                          ast.tokencode = TBaseType.rrw_sqlserver_offset_fetch;
                      }else{
                          // don't turn fetch to TBaseType.rrw_sqlserver_offset_fetch
                          //open row
                          //fetch row into @msg
                      }

                }
            }else if((ast.tokencode == TBaseType.rrw_exec)||(ast.tokencode == TBaseType.rrw_execute)){
                // search ;2 after execute
                // EXECUTE ApxSQL_Test_Triggers_Add;2  @TableName, @AddFlag
                int searchRange = 4;
                TSourceToken endTokenInSameLine = ast.searchTokenAtTheEndOfSameLine();
                if (endTokenInSameLine != null){
                    searchRange = endTokenInSameLine.posinlist - ast.posinlist;
                }
                TSourceToken st1 = ast.searchToken(';',searchRange);
                if (st1 != null){
                    TSourceToken numSt = st1.nextSolidToken();
                    if ((numSt != null)&&(numSt.tokentype == ETokenType.ttnumber)){
                        st1.tokencode = TBaseType.rrw_sqlserver_semicolon_module_number;
                        st1.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_sqlserver_trim){
                TSourceToken st1 =  ast.nextSolidToken();
                if (st1 != null){
                    if (st1.tokencode == '('){

                    }else{
                        ast.tokencode = TBaseType.ident; // change trim keyword to identifier if not followed by (), trim()
                    }
                }
            }

            if (dbVendor == EDbVendor.dbvopenedge){
                if (ast.tokencode == TBaseType.rrw_order){
                        TSourceToken st1 = ast.searchToken(TBaseType.rrw_by,1);
                        if (st1 == null) {
                            ast.tokencode = TBaseType.ident;
                        }
                }else if (ast.tokencode == TBaseType.rrw_with){
                        TSourceToken st1 = ast.searchToken(TBaseType.rrw_check,1);
                        if (st1 != null) {
                            ast.tokencode = TBaseType.rrw_openedge_with_check;
                        }
                }
            }

            if ( gst == EFindSqlStateType.ststoredprocedurebody )
            {
                if ( !(
                        (ast.tokencode == TBaseType.rrw_go)
                                || (ast.tokencode == TBaseType.rrw_create)
                                || (ast.tokencode == TBaseType.rrw_alter) )
                        )
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                    continue;
                }
            }

            TCustomSqlStatement lcnextsqlstatement =  sqlcmds.issql(ast, gst, gcurrentsqlstatement);

            switch(gst){
                case sterror:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        gst = EFindSqlStateType.stsql;
                    }
                    else if ( (ast.tokentype == ETokenType.ttsemicolon) )
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                }
                case stnormal :
                {
                    if ( (ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if ( TBaseType.assigned(gcurrentsqlstatement) )
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ( TBaseType.assigned(lcprevsolidtoken) && (ast.tokentype == ETokenType.ttsemicolon) )
                        {
                            if ( lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                            else
                            {
                            }

                        }

                        continue;
                    }

                    gcurrentsqlstatement = lcnextsqlstatement; //isstoredprocedure(ast,dbvendor,gst,sqlstatements);

                    if ( TBaseType.assigned(gcurrentsqlstatement) )
                    {
                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                            //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case   sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstmssqlbegintry:
                            case sstmssqlbegincatch:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.sttrycatch;
                                lctrycatchlevel = 0;
                                break;
                            }
                            case sstmssqlgo:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            default:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stsql;
                                break;
                            }
                        }    // case
                    }
                    else
                    {
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stblock;
                        }
                        else
                        {
                            if ( sqlstatements.size() == 0 )
                            {
                                //first statement of mssql batch, treat it as exec sp
                                gst = EFindSqlStateType.stsql;
                                gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                                //tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
// todo need previous line need to be implemented
                                gcurrentsqlstatement.addtokentolist(ast);
                            }
                            else if ( sqlstatements.get(sqlstatements.size() -1).sqlstatementtype == ESqlStatementType.sstmssqlgo )
                            {
                                // prev sql is go, treat it as exec sp
                                gst = EFindSqlStateType.stsql;
                                gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                                //todo need to be implemented: tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
                                gcurrentsqlstatement.addtokentolist(ast);
                            }
                            else
                            {
                            }
                        }
                    }


                    if (  !TBaseType.assigned(gcurrentsqlstatement)  ) //error tokentext found
                    {


                        this.syntaxErrors.add( new TSyntaxError(ast.getAstext(),ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));


                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    break;
                }
                case stblock:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else
                        {
                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.stblock )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            // { [distributed] transaxtion/trans statement
                            // { dialog [ conversation ]
                            // { conversation timer
                            //  doesn't start block ({ .. })
                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( ! ( (lcnextsolidtoken.tokencode ==  TBaseType.rrw_sqlserver_tran)
                                        || (lcnextsolidtoken.tokencode ==  TBaseType.rrw_sqlserver_transaction)
                                        || (lcnextsolidtoken.tokencode ==  TBaseType.rrw_sqlserver_distributed)
                                        || (lcnextsolidtoken.tokencode ==  TBaseType.rrw_sqlserver_dialog)
                                        || (lcnextsolidtoken.tokencode ==  TBaseType.rrw_sqlserver_conversation)
                                ) )
                                    lcblocklevel++;
                            }
                            else
                                lcblocklevel++;

                        }
                        else if ( ast.tokencode == TBaseType.rrw_case )  // case ... }
                            lcblocklevel++;
                        else if ( ast.tokencode == TBaseType.rrw_end )
                        {

                            lcisendconversation = false;


                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( lcnextsolidtoken.tokencode ==  TBaseType.rrw_sqlserver_conversation)
                                    lcisendconversation = true; // } conversation statement
                            }


                            if ( ! lcisendconversation )
                            {

                                if ( lcblocklevel == 0 )
                                {
                                    if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                                    {
                                        if ( TBaseType.assigned(lcnextsolidtoken) )
                                        {
                                            if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                            {
                                                // { .. } else
                                                gst = EFindSqlStateType.stsql;
                                            }
                                            else if ( lcnextsolidtoken.tokentype == ETokenType.ttsemicolon )
                                            {
                                                lcnnextsolidtoken = sourcetokenlist.nextsolidtoken(lcnextsolidtoken.posinlist,1,false);
                                                if ( TBaseType.assigned(lcnnextsolidtoken) )
                                                {
                                                    if ( lcnnextsolidtoken.tokencode == TBaseType.rrw_else )
                                                    {
                                                        // { .. } else
                                                        gst = EFindSqlStateType.stsql;
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    if ( gst != EFindSqlStateType.stsql )
                                    {
                                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                                        gst = EFindSqlStateType.stnormal;
                                    }

                                }
                                else
                                {
                                    lcblocklevel--;
                                }

                            }
                        }
                    }
                    break;
                }
                case sttrycatch:
                {

                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else
                        {
                            if (
                                    (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry)
                                            ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                                    )
                                lctrycatchlevel++;
                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.sttrycatch )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( (ast.tokencode == TBaseType.rrw_try) ||
                                (ast.tokencode == TBaseType.rrw_catch) )
                        {
                            // lcprevsolidtoken = sourcetokenlist.solidtokenbefore(i);
                            if ( TBaseType.assigned(lcprevsolidtoken) )
                            {
                                if ( lcprevsolidtoken.tokencode == TBaseType.rrw_end )
                                {
                                    if ( lctrycatchlevel == 0 )
                                    {
                                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                                        gst = EFindSqlStateType.stnormal;
                                    }
                                    else
                                        lctrycatchlevel--;
                                }
                            }
                        }
                    }
                    break;
                }
                case stprocedureWithReturn:
                {
                    // found return statement in create procedure/function, which means can be ended with a semicolon

                    if ( TBaseType.assigned(lcnextsqlstatement) ) {
                        if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo) {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                            break;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);
                    if ((gst == EFindSqlStateType.stprocedureWithReturn) && (ast.tokentype == ETokenType.ttsemicolon)){

                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    break;
                }
                case stsql :
                {
                    if ( (ast.tokentype == ETokenType.ttsemicolon)   )
                    {
                        lcstillinsql = false;
                        if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                        {
                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                {
                                    // if ( expr stmt; else
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    lcstillinsql = true;
                                }

                            }
                        }

                        if (  !lcstillinsql )
                        {
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gcurrentsqlstatement.semicolonended = ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }

                    }
                    else if ( TBaseType.assigned(lcnextsqlstatement) )
                    {

                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                            continue;
                        }

                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlif:
                            case sstmssqlwhile:
                            {
                                if ((lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                                        ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry))
                                {
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    gst = EFindSqlStateType.stblock;
                                    lcblocklevel = 1;
                                    lcnextsqlstatement = null;
                                    continue;

                                }
                                // if ( || while only contain one sql(not closed by {/} pair)
                                // so will still in if ( || while statement
                                else if ( gcurrentsqlstatement.dummytag == 1 )
                                {
                                    // if ( cond ^stmt nextstmt (^ stands for current pos)
                                    gcurrentsqlstatement.addtokentolist(ast);

                                    if ( (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                                            )
                                        gcurrentsqlstatement.dummytag = 1;
                                    else
                                        gcurrentsqlstatement.dummytag = 0;


                                    lcnextsqlstatement = null;
                                    continue;
                                }else{
                                    // 在 if statement 中，已经遇见过第一个sql语句，现在是碰到第二个，正常情况下这个第二个
                                    //IF EXISTS (SELECT * FROM sysobjects WHERE id = OBJECT_ID(N'[DF_TEST]') AND type = 'D')
                                    //ALTER TABLE test DROP CONSTRAINT IF EXISTS DF_TEST;

                                    if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif){
                                        // 上例中的第二个 IF EXISTS 不是真正的 sql， 忽略它
                                        if ((ast.nextSolidToken() != null)&&(ast.nextSolidToken().tokencode == TBaseType.rrw_sqlserver_exists)){
                                            gcurrentsqlstatement.addtokentolist(ast);
                                            lcnextsqlstatement = null;
                                            continue;
                                        }
                                    }
                                }
                                break;
                            }//
                            case sstmssqlalterqueue:
                            {
                                if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlexec )
                                {
                                    // execute can't be used to delimite alter queue
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    lcnextsqlstatement = null;
                                    continue;

                                }
                                break;
                            }
                            case sstmssqlcreateschema:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                lcnextsqlstatement = null;
                                continue;
                            }
                        }//case

                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);

                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                            //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstmssqlbegintry:
                            case sstmssqlbegincatch:
                            {
                                gst = EFindSqlStateType.sttrycatch;
                                lctrycatchlevel = 0;
                                break;
                            }
                            case sstmssqlgo:
                            {
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            default:
                            {
                                gst = EFindSqlStateType.stsql;
                                break;
                            }
                        }    // case

                    }//TBaseType.assigned(lcnextsqlstatement)
                    else if ( (ast.tokencode == TBaseType.rrw_begin) )
                    {
                        if ( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                                )
                        {
                            // start block of if ( || while statement
                            gst = EFindSqlStateType.stblock;
                            lcblocklevel = 0;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                        else if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqldeclare )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stblock;
                        }
                        else
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_case) ){
                        case_end_nest++;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_end) ){
                        if (case_end_nest > 0){
                            case_end_nest--;
                        }
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_else) )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        // if ( cond stmt ^else stmt
                        if (( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                        ) && (case_end_nest == 0))
                            gcurrentsqlstatement.dummytag = 1; // reduce to 1 while stmt after else is found: if ( cond stmt ^else stmt
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    break;
                }
                case ststoredprocedure:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else if (lcnextsqlstatement.sqlstatementtype == sstmssqlreturn)
                        {
                            gst = EFindSqlStateType.stprocedureWithReturn;
                            gcurrentsqlstatement.addtokentolist(ast);
                            lcnextsqlstatement = null;
                        }
                        else
                        {
                            gst = EFindSqlStateType.ststoredprocedurebody;
                            gcurrentsqlstatement.addtokentolist(ast);

                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.ststoredprocedure )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            gst = EFindSqlStateType.stblock;
                        }
                    }
                    break;
                } //stplsql
                case ststoredprocedurebody:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        switch(lcnextsqlstatement.sqlstatementtype){    //
                            case sstmssqlgo:
                            {
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                            //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstcreateview:
                            case sstcreatetable:
                            {

                                boolean readForNewStmt = false;
                                TSourceToken st1 = ast.searchToken(';',-1);
                                if (st1 != null) {
                                    TSourceToken st2 = ast.searchToken(TBaseType.rrw_end,-2);
                                    if (st2 != null){
                                        readForNewStmt = true;
                                    }
                                }

                                if (readForNewStmt){
                                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                                    gcurrentsqlstatement = lcnextsqlstatement;
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    gst = EFindSqlStateType.stsql;
                                }else{
                                    lcnextsqlstatement = null;
                                }
                                break;
                            }
                            case sstmssqlDropSecurityPolicy:
                            case sstmssqlAlterSecurityPolicy:
                            case sstmssqlCreateSecurityPolicy:
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stsql;

                                    break;
                            default:
                            {
                                lcnextsqlstatement = null;
                                break;
                            }
                        }//case
                    }

                    if ( gst == EFindSqlStateType.ststoredprocedurebody )
                        gcurrentsqlstatement.addtokentolist(ast);
                }
                break;
            } //case
        } //for


        //last statement
        if ( TBaseType.assigned(gcurrentsqlstatement) &&  (gst != EFindSqlStateType.stnormal))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement,true);
        }

        return errorcount;

    }


    private TCustomSqlStatement startDaxStmt(TSourceToken currToken,TCustomSqlStatement currStmt){
        TCustomSqlStatement newStmt = null;
        if (currToken == null) return null;
        if ((currToken.tokencode == '=')&&(currToken.isFirstTokenOfLine())){
            currToken.tokencode = TBaseType.equal_start_expr;
            newStmt = new TDaxExprStmt(EDbVendor.dbvdax);
        }else if ((currToken.tokencode == TBaseType.rrw_dax_define)&&(currToken.isFirstTokenOfLine())){
            newStmt = new TDaxEvaluateStmt(EDbVendor.dbvdax);
            ((TDaxEvaluateStmt)newStmt).setStartWithDefine(true);
        }else if ((currToken.tokencode == TBaseType.rrw_dax_evaluate)&&(currToken.isFirstTokenOfLine())){
            if ((currStmt != null)&&(currStmt instanceof TDaxEvaluateStmt)){
                TDaxEvaluateStmt tmp = (TDaxEvaluateStmt)currStmt;
                if (tmp.isStartWithDefine()) return  null;
            }
            newStmt = new TDaxEvaluateStmt(EDbVendor.dbvdax);
        }

        if (newStmt == null){
            // let's check is this the first token of query
            boolean isFirst = currToken.isFirstTokenOfLine();
            TSourceToken prevToken = currToken.prevSolidToken();
            if ((isFirst)&&(prevToken == null)){
                newStmt = new TDaxExprStmt(EDbVendor.dbvdax);
            }
        }
        return newStmt;
    }

int dogetrawsqlstatements(){

    sqlstatements.clear();
    if (sourcetokenlist.size() == 0) return -1;

    switch(dbVendor){
        case dbvpostgresql:{
            return dopostgresqlgetrawsqlstatements();
        }
        case dbvclickhouse:{
            return doclickhousegetrawsqlstatements();
        }
        default:{
            return domssqlgetrawsqlstatements();
        }
    }

}

    /**
     *  separates the SQL statements in the input SQL script without doing syntax check.
     *  <p></p>
     *  Use the {@link #getSqlstatements()} method to get the list of SQL statements.
     *  The SQL statement object is the instance of the sub-class of {@link TCustomSqlStatement}, get SQL statement type
     *  via the {@link TCustomSqlStatement#sqlstatementtype} field, get string representation of
     *  each SQL statement via the {@link TCustomSqlStatement#toString} method.
     *  <p></p>
     *  All source tokens in this SQL statement
     *  is available by using {@link TCustomSqlStatement#sourcetokenlist} filed.
     *  Since no parse tree is built by calling this method, no further detailed information about the SQL statement is available.
     *
      * @return 0 if get SQL statements successfully
     */
public int getrawsqlstatements(){
    // Check for vendor parser delegation (MSSQL, etc.)
    // This ensures raw-split logic is maintained in only one place (vendor parser)
    SqlParser vp = getOrCreateVendorParser();
    if (vp != null) {
        return doDelegatedRawParse(vp);
    }

    // Legacy path for non-delegated vendors
    int ret = readsql();
    if (ret != 0) return ret;
    dosqltexttotokenlist();

    return dogetrawsqlstatements();
}

    /**
     *  turns the input SQL into a sequence of token which is the
     *  basic lexis element of SQL syntax. Token is categorized as keyword, identifier,
     *  number, operator, whitespace and other types. All source tokens can be fetched
     *  via the {@link #getSourcetokenlist()} method.
     *
     */
public void tokenizeSqltext(){
    // Check for vendor parser delegation (MSSQL, etc.)
    // This ensures tokenization logic is maintained in only one place (vendor parser)
    SqlParser vp = getOrCreateVendorParser();
    if (vp != null) {
        doDelegatedTokenize(vp);
        return;
    }

    // Legacy path for non-delegated vendors
    getFlexer();
    readsql();
    dosqltexttotokenlist();
}

/**
 * Delegate tokenization to vendor parser and backfill results.
 * Similar to doDelegatedRawParse but only performs tokenization.
 */
private void doDelegatedTokenize(SqlParser vendorParser) {
    // Build context for vendor parser
    ParserContext context = buildContext();

    // Delegate tokenization to vendor parser
    SqlParseResult tokenResult = vendorParser.tokenize(context);

    // Copy results back to TGSqlParser fields for backward compatibility
    if (tokenResult.getSourceTokenList() != null) {
        this.sourcetokenlist = tokenResult.getSourceTokenList();
    }
    if (tokenResult.getLexer() != null) {
        this.flexer = tokenResult.getLexer();
    }
}



void findAllSyntaxErrorsInPlsql(TCustomSqlStatement psql){
    if (psql.getErrorCount() > 0){
        copyerrormsg(psql);
    }

    for (int k=0;k<psql.getStatements().size();k++){
        findAllSyntaxErrorsInPlsql(psql.getStatements().get(k));
    }

}

private TSQLEnv sqlEnv = null;

    /**
     * SQL environment includes the database metadata such as procedure, function, trigger, table and etc.
     *
     * In order to link column to table correctly without connecting to database,
     * we need to provide a class which implements {@link TSQLEnv} to TGSqlParser.
     * this class tells TGSqlParser the relationship between column and table.
     *
     * <p>Take this SQL for example:
     * <pre>
     * SELECT Quantity,b.Time,c.Description
     * FROM
     * (SELECT ID2,Time FROM bTab) b
     * INNER JOIN aTab a on a.ID=b.ID
     * INNER JOIN cTab c on a.ID=c.ID
     * </pre>
     *
     * <p>General SQL Parser can build relationship between column: ID2 and table: bTable
     * correctly without metadata information from database because there is only one table
     * in from clause. But it can't judge column: Quantity belong to table: aTab or cTab,
     * since no table alias was prefixed to column: Quantity. If no metadata provided,
     * General SQL Parser will link column: Quantity to the first valid table (here it is aTab)
     *
     * <p>If we create a class  TRealDatabaseSQLEnv implements {@link TSQLEnv},then
     *  {@link #setSqlEnv(TSQLEnv)}, General SQL Parser can take this advantage to create
     *  a correct relationship between column and tables.
     *
     *<pre>
     * class TSQLServerEnv extends TSQLEnv{
     *
     *  public TSQLServerEnv(){
     *          super(EDbVendor.dbvmssql);
     *          initSQLEnv();
     *        }
     *
     *    &#64;Override
     *    public void initSQLEnv() {
     *
     *          // add a new database: master
     *          TSQLCatalog sqlCatalog = createSQLCatalog("master");
     *          // add a new schema: dbo
     *          TSQLSchema sqlSchema = sqlCatalog.createSchema("dbo");
     *          //add a new table: aTab
     *          TSQLTable aTab = sqlSchema.createTable("aTab");
     *          aTab.addColumn("Quantity1");
     *
     *          //add a new table: bTab
     *          TSQLTable bTab = sqlSchema.createTable("bTab");
     *          bTab.addColumn("Quantity2");
     *
     *          //add a new table: cTab
     *          TSQLTable cTab = sqlSchema.createTable("cTab");
     *          cTab.addColumn("Quantity");
     *
     *    }
     * }
     * </pre>
     *
     * @return SQL environment
     */
    public TSQLEnv getSqlEnv() {
        return sqlEnv;
    }

    private boolean onlyNeedRawParseTree = false;

    public void setOnlyNeedRawParseTree(boolean onlyNeedRawParseTree) {
        this.onlyNeedRawParseTree = onlyNeedRawParseTree;
    }

    public void setSqlEnv(TSQLEnv sqlEnv) {
        this.sqlEnv = sqlEnv;
    }


     // Time tracking variables
     private boolean enableTimeLogging = false;
     private long rawSqlStatementsTime = 0;
     private long parsingTime = 0;
     private long semanticAnalysisTime = 0;
     private long interpreterTime = 0;
     
     /**
      * Enable or disable time logging for parser steps
      * @param enable true to enable time logging, false to disable
      */
     public void setEnableTimeLogging(boolean enable) {
         this.enableTimeLogging = enable;
     }
     
     /**
      * Check if time logging is enabled
      * @return true if time logging is enabled, false otherwise
      */
     public boolean isTimeLoggingEnabled() {
         return this.enableTimeLogging;
     }
     
     /**
      * Reset all accumulated time counters to zero
      */
     public void resetTimeCounters() {
         rawSqlStatementsTime = 0;
         parsingTime = 0;
         semanticAnalysisTime = 0;
         interpreterTime = 0;
     }
     
     /**
      * Get accumulated time spent getting raw SQL statements in milliseconds
      * @return time in milliseconds
      */
     public long getRawSqlStatementsTime() {
         return rawSqlStatementsTime;
     }
     
     /**
      * Get accumulated time spent parsing in milliseconds
      * @return time in milliseconds
      */
     public long getParsingTime() {
         return parsingTime;
     }
     
     /**
      * Get accumulated time spent on semantic analysis in milliseconds
      * @return time in milliseconds
      */
     public long getSemanticAnalysisTime() {
         return semanticAnalysisTime;
     }
     
     /**
      * Get accumulated time spent in interpreter in milliseconds
      * @return time in milliseconds
      */
     public long getInterpreterTime() {
         return interpreterTime;
     }
     
     /**
      * Get total accumulated time spent in all steps
      * @return total time in milliseconds
      */
     public long getTotalTime() {
         return rawSqlStatementsTime + parsingTime + semanticAnalysisTime + interpreterTime;
     }

    /**
     * Get or create the vendor-specific parser for delegation.
     * The parser is cached in vendorParser field and reused for subsequent calls.
     * This allows getFlexer() to lazily create the parser to access its lexer.
     *
     * @return vendor-specific SqlParser or null if not a delegated vendor
     * @since 3.2.0.0
     */
    private SqlParser getOrCreateVendorParser() {
        // Return cached parser if available
        if (vendorParser != null) {
            return vendorParser;
        }

        // Create new vendor parser based on database vendor
        switch (dbVendor) {
            case dbvmssql:
            case dbvazuresql:
                vendorParser = new gudusoft.gsqlparser.parser.MssqlSqlParser();
                break;
            case dbvmysql:
                vendorParser = new gudusoft.gsqlparser.parser.MySqlSqlParser();
                break;
            case dbvpostgresql:
                vendorParser = new gudusoft.gsqlparser.parser.PostgreSqlParser();
                break;
            case dbvoracle:
                vendorParser = new gudusoft.gsqlparser.parser.OracleSqlParser();
                break;
            case dbvbigquery:
                vendorParser = new gudusoft.gsqlparser.parser.BigQuerySqlParser();
                break;
            case dbvathena:
                vendorParser = new gudusoft.gsqlparser.parser.AthenaSqlParser();
                break;
            case dbvcouchbase:
                vendorParser = new gudusoft.gsqlparser.parser.CouchbaseSqlParser();
                break;
            case dbvdatabricks:
                vendorParser = new gudusoft.gsqlparser.parser.DatabricksSqlParser();
                break;
            case dbvdax:
                vendorParser = new gudusoft.gsqlparser.parser.DaxSqlParser();
                break;
            case dbvdb2:
                vendorParser = new gudusoft.gsqlparser.parser.Db2SqlParser();
                break;
            case dbvgaussdb:
                vendorParser = new gudusoft.gsqlparser.parser.GaussDbSqlParser();
                break;
            case dbvgreenplum:
                vendorParser = new gudusoft.gsqlparser.parser.GreenplumSqlParser();
                break;
            case dbvhive:
                vendorParser = new gudusoft.gsqlparser.parser.HiveSqlParser();
                break;
            case dbvhana:
                vendorParser = new gudusoft.gsqlparser.parser.HanaSqlParser();
                break;
            case dbvimpala:
                vendorParser = new gudusoft.gsqlparser.parser.ImpalaSqlParser();
                break;
            case dbvinformix:
                vendorParser = new gudusoft.gsqlparser.parser.InformixSqlParser();
                break;
            case dbvmdx:
                vendorParser = new gudusoft.gsqlparser.parser.MdxSqlParser();
                break;
            case dbvnetezza:
                vendorParser = new gudusoft.gsqlparser.parser.NetezzaSqlParser();
                break;
            case dbvodbc:
                vendorParser = new gudusoft.gsqlparser.parser.OdbcSqlParser();
                break;
            case dbvopenedge:
                vendorParser = new gudusoft.gsqlparser.parser.OpenEdgeSqlParser();
                break;
            case dbvpresto:
                vendorParser = new gudusoft.gsqlparser.parser.PrestoSqlParser();
                break;
            case dbvredshift:
                vendorParser = new gudusoft.gsqlparser.parser.RedshiftSqlParser();
                break;
            case dbvsnowflake:
                vendorParser = new gudusoft.gsqlparser.parser.SnowflakeSqlParser();
                break;
            case dbvsoql:
                vendorParser = new gudusoft.gsqlparser.parser.SoqlSqlParser();
                break;
            case dbvsparksql:
                vendorParser = new gudusoft.gsqlparser.parser.SparksqlSqlParser();
                break;
            case dbvsybase:
                vendorParser = new gudusoft.gsqlparser.parser.SybaseSqlParser();
                break;
            case dbvteradata:
                vendorParser = new gudusoft.gsqlparser.parser.TeradataSqlParser();
                break;
            case dbvvertica:
                vendorParser = new gudusoft.gsqlparser.parser.VerticaSqlParser();
                break;
            case dbvansi:
                vendorParser = new gudusoft.gsqlparser.parser.AnsiSqlParser();
                break;
            default:
                return null;
        }
        return vendorParser;
    }

    /**
     * Prepare this parser for reuse by clearing cached state.
     * This is useful when reusing a TGSqlParser instance (e.g., in TExecImmeStmt)
     * to avoid state pollution between parsing operations.
     *
     * Clears: vendorParser, sqlEnv, sqlfilename, and resets parsing options.
     *
     * @since 3.2.0.0
     */
    public void prepareForReuse() {
        // Clear cached vendor parser to force recreation
        this.vendorParser = null;
        // Clear SQL environment to avoid old schema information affecting new parse
        this.sqlEnv = null;
        // Clear filename in case it was set previously
        this.sqlfilename = null;
        // Reset parsing options to defaults
        this.isSinglePLBlock = false;
        // Clear statement list
        if (this.sqlstatements != null) {
            this.sqlstatements.clear();
        }
        // Clear error state
        this.syntaxErrors.clear();
    }

    /**
     * Build a ParserContext from current TGSqlParser state.
     * This creates an immutable context for delegation to vendor parsers.
     *
     * @return immutable ParserContext
     * @since 3.2.0.0
     */
    private ParserContext buildContext() {
        ParserContext.Builder builder = new ParserContext.Builder(this.dbVendor);

        // Set SQL text source (only one will be non-null or non-empty)
        // CRITICAL: Check for non-null AND non-empty because TGSqlParser initializes sqltext=""
        // which would cause vendor parser to use empty string instead of reading from file
        if (this.sqltext != null && !this.sqltext.isEmpty()) {
            builder.sqlText(this.sqltext);
        }
        if (this.sqlfilename != null && !this.sqlfilename.isEmpty()) {
            builder.sqlFilename(this.sqlfilename);
        }

        // Set charset
        if (this.sqlCharset != null) {
            builder.sqlCharset(this.sqlCharset);
        }

        // Set parsing options
        builder.enablePartialParsing(this.isEnablePartialParsing());
        builder.singlePLBlock(this.isSinglePLBlock);

        // Set gsqlparser reference for backward compatibility
        builder.gsqlparser(this);

        // Set SQL environment if available
        if (this.sqlEnv != null) {
            builder.sqlEnv(this.sqlEnv);
        }

        // Set token handle for callback during tokenization
        if (this.tokenHandle != null) {
            builder.tokenHandle(this.tokenHandle);
        }

        return builder.build();
    }

     /**
      * Returns time statistics as a formatted string
      * @return formatted string with time statistics
      */
     public String getTimeStatistics() {
         if (!enableTimeLogging) {
             return "Time logging is disabled";
         }
         
         StringBuilder sb = new StringBuilder();
         sb.append(String.format("Time statistics for TGSqlParser version: %s, released at: %s, dbvendor: %s:\n", TBaseType.versionid,TBaseType.releaseDate, this.dbVendor));
         sb.append(String.format("1. Raw SQL statements: %d ms (%.2f%%)\n", 
                 rawSqlStatementsTime,
                 getTotalTime() > 0 ? 100.0 * rawSqlStatementsTime / getTotalTime() : 0));
         sb.append(String.format("2. Parsing: %d ms (%.2f%%)\n", 
                 parsingTime,
                 getTotalTime() > 0 ? 100.0 * parsingTime / getTotalTime() : 0));
         sb.append(String.format("3. Semantic analysis: %d ms (%.2f%%)\n", 
                 semanticAnalysisTime,
                 getTotalTime() > 0 ? 100.0 * semanticAnalysisTime / getTotalTime() : 0));
         sb.append(String.format("4. Interpreter: %d ms (%.2f%%)\n", 
                 interpreterTime,
                 getTotalTime() > 0 ? 100.0 * interpreterTime / getTotalTime() : 0));
         sb.append(String.format("Total: %d ms", getTotalTime()));
         return sb.toString();
     }

    /**
     * Delegate only tokenization and raw statement extraction to vendor parser.
     * The actual AST parsing is done by the common parsing loop in doparse().
     * This architecture ensures that:
     * 1. Vendor-specific tokenization/lexing is handled by vendor parser
     * 2. Raw statement extraction (splitting SQL text into statements) is vendor-specific
     * 3. AST parsing (building parse tree for each statement) uses common code path
     *
     * @param vendorParser the vendor-specific parser to use
     * @return 0 if successful, non-zero on error
     */
    private int doDelegatedRawParse(SqlParser vendorParser) {
        long phaseStart, phaseEnd;

        // Build context for vendor parser
        ParserContext context = buildContext();

        // Delegate only tokenization + raw statement extraction to vendor parser
        phaseStart = enableTimeLogging ? System.currentTimeMillis() : 0;
        SqlParseResult rawResult = vendorParser.getrawsqlstatements(context);
        if (enableTimeLogging) {
            phaseEnd = System.currentTimeMillis();
            rawSqlStatementsTime += (phaseEnd - phaseStart);
        }

        // Copy results back to TGSqlParser fields for backward compatibility
        if (rawResult.getSourceTokenList() != null) {
            this.sourcetokenlist = rawResult.getSourceTokenList();
            this.sourcetokenlist.setGsqlparser(this);
        }
        if (rawResult.getLexer() != null) {
            this.flexer = rawResult.getLexer();
        }
        // Copy parser from vendor parser result - needed for parsestatement()
        if (rawResult.getParser() != null) {
            this.fparser = rawResult.getParser();
            // CRITICAL: Set gsqlparser on NodeFactory for correct AST construction
            this.fparser.getNf().setGsqlParser(this);
        }
        // Copy secondary parser (for Oracle PL/SQL) and set its NodeFactory
        if (rawResult.getSecondaryParser() != null) {
            this.fplsqlparser = rawResult.getSecondaryParser();
            // CRITICAL: Also set gsqlparser on secondary parser's NodeFactory
            // This is needed for Oracle PL/SQL statements which use fplsqlparser for parsing
            this.fplsqlparser.getNf().setGsqlParser(this);
        }
        if (rawResult.getSqlStatements() != null) {
            this.sqlstatements = rawResult.getSqlStatements();
        }

        // Copy any tokenization errors
        if (rawResult.getSyntaxErrors() != null) {
            for (TSyntaxError error : rawResult.getSyntaxErrors()) {
                this.syntaxErrors.add(error);
            }
        }

        return rawResult.getErrorCode();
    }

     int doparse() {
         int j;
         long startTime, endTime;
         boolean useDelegatedRawParse = false;

         // 1. Get raw SQL statements
         // For delegated vendors, use vendor parser for tokenization + raw extraction
         // The parsing loop below is common for all vendors
         SqlParser vp = getOrCreateVendorParser();
         if (vp != null) {
             int ret = doDelegatedRawParse(vp);
             if (ret != 0) {
                 // Tokenization/extraction failed, return error
                 return ret;
             }
             useDelegatedRawParse = true;
             // Continue to common parsing loop below
         }

         // Legacy path: get raw statements for non-delegated vendors
         if (!useDelegatedRawParse) {
             startTime = enableTimeLogging ? System.currentTimeMillis() : 0;

             int ret = getrawsqlstatements();

             if (enableTimeLogging) {
                 endTime = System.currentTimeMillis();
                 rawSqlStatementsTime += (endTime - startTime);
             }
         }

         boolean isPushGloablStack = false;
         //if (ret != 0) return ret;
         TFrame firstFrame = null;
 
         globalContext = new TContext();
 
         if (this.sqlEnv == null) {
             this.sqlEnv = new TSQLEnv(this.dbVendor) {
                 @Override
                 public void initSQLEnv() {
                 }
             };
             // this.sqlEnv.setEnableGetMetadataFromDDL(false);
         }
         globalContext.setSqlEnv(this.sqlEnv, this.sqlstatements);
 
         //TStackFrame stackFrame = new TStackFrame(new TGlobalScope());
         if (getFrameStack().size() == 0) { // stack passed from outside gsqlparser will contain frames
             //getFrameStack().push(new TStackFrame(new TGlobalScope()));
             TGlobalScope globalScope = new TGlobalScope();
             globalScope.resetCurrentStmtIndex();
 
             globalScope.setSqlEnv(this.sqlEnv);
             firstFrame = new TFrame(globalScope);
             firstFrame.pushMeToStack(getFrameStack());
             isPushGloablStack = true;
         }
 
         // 2. start parsing
         startTime = enableTimeLogging ? System.currentTimeMillis() : 0;
         
         for (int i = 0; i < sqlstatements.size(); i++) {
             sqlstatements.getRawSql(i).setFrameStack(frameStack);
             j = sqlstatements.getRawSql(i).parsestatement(null, false, onlyNeedRawParseTree);

             // NOTE: No need for setGsqlparserRecursively() here for delegated vendors.
             // The gsqlparser reference is already set on all AST nodes through two mechanisms:
             // 1. doDelegatedRawParse() sets fparser.getNf().setGsqlParser(this) and
             //    fplsqlparser.getNf().setGsqlParser(this) BEFORE parsestatement() is called
             // 2. TNodeFactory.createNode() propagates gsqlparser to every node it creates
             // The recursive walk was causing 4-5x performance degradation for Oracle parsing
             // by redundantly traversing the entire AST after parsing.

             TCustomSqlStatement sql0 = null;
             if (sqlstatements.get(i).isoracleplsql()) {
                 // check syntax error in select/insert statement inside plsql
                 sql0 = sqlstatements.get(i);
                 findAllSyntaxErrorsInPlsql(sql0);
             }
 
             boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
 
             if (doRecover && ((j != 0) || (sqlstatements.get(i).getErrorCount() > 0))) {
                 if (((sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreatetable)
                         || ((sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreateindex) && (dbVendor != EDbVendor.dbvcouchbase))
                         ) && (!TBaseType.c_createTableStrictParsing)
                 ) {
                     // only parse main body of create table,
                     TCustomSqlStatement errorSqlStatement = (TCustomSqlStatement) sqlstatements.get(i);
 
                     int nested = 0;
                     boolean isIgnore = false, isFoundIgnoreToken = false;
                     TSourceToken firstIgnoreToken = null;
                     for (int k = 0; k < errorSqlStatement.sourcetokenlist.size(); k++) {
                         TSourceToken st = errorSqlStatement.sourcetokenlist.get(k);
                         if (isIgnore) {
                             if (st.issolidtoken() && (st.tokencode != ';')) {
                                 isFoundIgnoreToken = true;
                                 if (firstIgnoreToken == null) {
                                     firstIgnoreToken = st;
                                 }
                             }
                             if (st.tokencode != ';') {
                                 st.tokencode = TBaseType.sqlpluscmd;
                             }
                             continue;
                         }
                         if (st.tokencode == (int) ')') {
                             nested--;
                             if (nested == 0) {
                                 //let's check is next token is
                                 // as ( select
                                 boolean isSelect = false;
                                 TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
                                 if (st1 != null) {
                                     TSourceToken st2 = st.searchToken((int) '(', 2);
                                     if (st2 != null) {
                                         TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
                                         isSelect = (st3 != null);
                                     }
                                 }
                                 if (!isSelect) isIgnore = true;
                             }
                         }
                         if ((st.tokencode == (int) '(') || (st.tokencode == TBaseType.left_parenthesis_2)) {
                             nested++;
                         }
                     }
 
                     if ((dbVendor == EDbVendor.dbvoracle) && ((firstIgnoreToken != null) && (!TBaseType.searchOracleTablePros(firstIgnoreToken.toString())))) {
                         // if it is not the valid Oracle table properties option, let raise the error.
                         isFoundIgnoreToken = false;
                     }
                     if (isFoundIgnoreToken) {
                         errorSqlStatement.clearError();
                         j = sqlstatements.get(i).parsestatement(null, false);
                     }
                 }
 
                 if (((sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreatetrigger)
                         || (sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreatefunction)
                         || (sqlstatements.get(i).sqlstatementtype == sstcreateprocedure)
                         ) && (dbVendor == EDbVendor.dbvdb2)) { 
                     // db2 can handle Oracle pl/sql code, so we give it a try here
                     TCustomSqlStatement stmt = sqlstatements.get(i);
                     StringBuffer stmtStr = new StringBuffer(1024);
                     for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
                         stmtStr.append(stmt.sourcetokenlist.get(k).getAstext());
                     }
 
                     TGSqlParser lc_sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
                     lc_sqlparser.sqltext = stmtStr.toString();
                     int iRet = lc_sqlparser.parse();
                     if (iRet == 0) {
                         sqlstatements.remove(i);
                         sqlstatements.add(i, lc_sqlparser.sqlstatements.get(0));
                         continue;
                     }
                 }
             }
 
             if ((j != 0) || (sqlstatements.get(i).getErrorCount() > 0)) {
                 copyerrormsg(sqlstatements.get(i));
 
                 if ((isEnablePartialParsing()) && (dbVendor == EDbVendor.dbvsybase) && (sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstmssqlcreateprocedure)) {
                     TMssqlCreateProcedure createProcedure = (TMssqlCreateProcedure) sqlstatements.get(i);
 
                     StringBuffer storedProcedure = new StringBuffer(1024);
                     boolean ASKeyword = false;
                     for (int k = 0; k < createProcedure.sourcetokenlist.size(); k++) {
                         if ((!ASKeyword) && (createProcedure.sourcetokenlist.get(k).tokencode == TBaseType.rrw_as)) {
                             ASKeyword = true;
                             continue;
                         }
                         if (ASKeyword) {
                             storedProcedure.append(createProcedure.sourcetokenlist.get(k).getAstext());
                         }
                     }
 
                     TGSqlParser lc_sqlparser = new TGSqlParser(dbVendor);
                     lc_sqlparser.sqltext = storedProcedure.toString();
                     lc_sqlparser.parse();
                     for (int k = 0; k < lc_sqlparser.sqlstatements.size(); k++) {
                         createProcedure.getBodyStatements().add(lc_sqlparser.sqlstatements.get(k));
                     }
                 }
             }
 
             // fire SQL Statement handle event if set, if return true, stop parsing
             if (sqlStatementHandle != null) {
                 if (sqlStatementHandle.processSQLStatement(sqlstatements.get(i), this)) break;
             }
         }
         
         if (enableTimeLogging) {
             endTime = System.currentTimeMillis();
             parsingTime += (endTime - startTime);
         }
 
         if (isPushGloablStack) {
             firstFrame.popMeFromStack(getFrameStack());
         }
 
         // 3. start semantic analysis
         startTime = enableTimeLogging ? System.currentTimeMillis() : 0;

         // Reset resolver2 for each parse
         this.resolver2 = null;

         // Determine effective resolver type (instance setting takes precedence over TBaseType)
         EResolverType effectiveResolverType = this.resolverType;
         if (effectiveResolverType == EResolverType.DEFAULT) {
             // Fall back to TBaseType settings for backward compatibility
             if (TBaseType.isEnableResolver2()) {
                 effectiveResolverType = EResolverType.RESOLVER2;
             } else if (TBaseType.isEnableResolver()) {
                 effectiveResolverType = EResolverType.RESOLVER;
             } else {
                 effectiveResolverType = EResolverType.NONE;
             }
         }

         // Run the appropriate resolver based on effective type
         if (getErrorCount() == 0) {
             switch (effectiveResolverType) {
                 case RESOLVER:
                     TSQLResolver resolver = new TSQLResolver(globalContext, sqlstatements);
                     if (!resolver.resolve()) {
                         // Handle resolution errors
                         // addErrors(resolver.getLog().getErrors());
                     }
                     break;

                 case RESOLVER2:
                     // Use provided config or create default
                     TSQLResolverConfig config = this.resolver2Config;
                     if (config == null) {
                         config = new TSQLResolverConfig();
                     }
                     // Always set vendor on config (needed for vendor-specific resolution like struct-field fallback)
                     config.setVendor(dbVendor);
                     // Pass null as globalContext to match original TSQLResolver2 behavior
                     this.resolver2 = new TSQLResolver2(null, sqlstatements, config);
                     // Pass sqlEnv to resolver2 for metadata lookup
                     if (this.sqlEnv != null) {
                         this.resolver2.setSqlEnv(this.sqlEnv);
                     }
                     if (!this.resolver2.resolve()) {
                         // Handle resolution errors if needed
                     }
                     break;

                 case NONE:
                 default:
                     // No resolution
                     break;
             }
         }

         if (enableTimeLogging) {
             endTime = System.currentTimeMillis();
             semanticAnalysisTime += (endTime - startTime);
         }
 
         // 4. start interpreter
         startTime = enableTimeLogging ? System.currentTimeMillis() : 0;
         
         if ((TBaseType.ENABLE_INTERPRETER) && (getErrorCount() == 0)) {
             TLog.clearLogs();
             TGlobalScope globalScope = new TGlobalScope(sqlEnv);
             TLog.enableInterpreterLogOnly();
 
             TASTEvaluator astEvaluator = new TASTEvaluator(this.sqlstatements, globalScope);
             astEvaluator.eval();
         }
         
         if (enableTimeLogging) {
             endTime = System.currentTimeMillis();
             interpreterTime += (endTime - startTime);
         }
 
         return getErrorCount();
     }

     
void copyerrormsg(TCustomSqlStatement sql){
    for (int i = 0; i<sql.getSyntaxErrors().size(); i++){
            this.syntaxErrors.add(new TSyntaxError( (TSyntaxError)sql.getSyntaxErrors().get(i) ));
    }
//    for (int i = 0; i<sql.getSyntaxHints().size(); i++){
//            this.syntaxHints.add(new TSyntaxError( (TSyntaxError)sql.getSyntaxHints().get(i) ));
//    }
}

private static String calculateLicenseKey(boolean ignoreMachineId){

    if (userName == null) return null;
    if (machineId == null) return null;

    byte[] bytesOfMessage=null;
    String licenseStr = "I love sql pretty printer, yeah!"+userName.toLowerCase();
    if (!ignoreMachineId){
        licenseStr += machineId.toLowerCase();
    }
    try {
        bytesOfMessage = licenseStr.getBytes("UTF-8");
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    }

    MessageDigest md = null;
    try {
        md = MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
        e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    }
    byte[] digest = md.digest(bytesOfMessage);

    return null;//HardwareBinder.getHex(digest);
}

private static boolean validateLicense(){

    int ret = 0;
    return ret == 0;

}

private static boolean check_license_time(){

    boolean  ret = false;

    String toDate = TBaseType.license_expired_date;

    DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT);
    Calendar currDtCal = Calendar.getInstance();

    // Zero out the hour, minute, second, and millisecond
    currDtCal.set(Calendar.HOUR_OF_DAY, 0);
    currDtCal.set(Calendar.MINUTE, 0);
    currDtCal.set(Calendar.SECOND, 0);
    currDtCal.set(Calendar.MILLISECOND, 0);

    Date currDt = currDtCal.getTime();

    Date toDt;
    try {
        toDt = df.parse(toDate);
    } catch (ParseException e) {
        toDt = null;
        // Print some error message back to the user
    }

    if (toDt != null){
        int results = toDt.compareTo(currDt);
        ret = results > 0;
    }

    return ret;
}

    public void setTeradataUtilityType(TeradataUtilityType teradataUtilityType) {
        if ((this.getFlexer() != null) && (this.getFlexer() instanceof TLexerTeradata)){
            ((TLexerTeradata)this.getFlexer()).setTeradataUtilityType(teradataUtilityType);
        }
    }
    
    /**
     * Dispose of parser resources and clean up references.
     * 
     * <p>This method releases internal resources used by the parser.
     * After calling this method, the parser should not be used further.</p>
     * 
     * <p>If a ManagedSourceBuffer was set, the source text remains in the buffer
     * so that tokens can still access it. The user is responsible for calling 
     * {@link ManagedSourceBuffer#release()} when all tokens are no longer needed.</p>
     * 
     * <p>Note: Tokens created by this parser will remain usable after dispose()
     * if a ManagedSourceBuffer was used, as they reference the buffer by ID
     * rather than holding direct parser references.</p>
     * 
     * @since 3.1.0.9
     */
    public void dispose() {
        // Note: We intentionally do NOT remove the source from managed buffer
        // This allows tokens to remain usable after parser disposal
        // The user must call buffer.release() when done with all tokens
        
        // Clear references to help GC
        flexer = null;
        fparser = null;
        fplsqlparser = null;
        finputstream = null;
        sqlInputStream = null;
        gcurrentsqlstatement = null;
        nextStmt = null;
        sqlstatements = null;
        sourcetokenlist = null;
        syntaxErrors = null;
    }

}