package gudusoft.gsqlparser;

import gudusoft.gsqlparser.compiler.TASTEvaluator;
import gudusoft.gsqlparser.compiler.TContext;
import gudusoft.gsqlparser.compiler.TGlobalScope;
import gudusoft.gsqlparser.compiler.TFrame;
import gudusoft.gsqlparser.nodes.*;
import gudusoft.gsqlparser.nodes.teradata.TTeradataHelper;
import gudusoft.gsqlparser.resolver.*;
import gudusoft.gsqlparser.sqlenv.TSQLEnv;
import gudusoft.gsqlparser.stmt.*;
import gudusoft.gsqlparser.stmt.dax.TDaxEvaluateStmt;
import gudusoft.gsqlparser.stmt.dax.TDaxExprStmt;
import gudusoft.gsqlparser.stmt.greenplum.TSlashCommand;
import gudusoft.gsqlparser.stmt.mssql.TMssqlBlock;
import gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure;
import gudusoft.gsqlparser.stmt.mssql.TMssqlExecute;
import gudusoft.gsqlparser.stmt.mysql.TMySQLSource;
import gudusoft.gsqlparser.stmt.oracle.TPlsqlCreatePackage;
import gudusoft.gsqlparser.stmt.oracle.TSqlplusCmdStatement;
import gudusoft.gsqlparser.stmt.snowflake.TCreateTaskStmt;
import gudusoft.gsqlparser.stmt.teradata.TTeradataBTEQCmd;
import gudusoft.gsqlparser.stmt.teradata.TTeradataFastExportCmd;
import gudusoft.gsqlparser.stmt.teradata.TTeradataFastLoadCmd;
import gudusoft.gsqlparser.stmt.teradata.TTeradataMultiLoadCmd;
import gudusoft.gsqlparser.stmt.teradata.utilities.BteqCmdType;
import gudusoft.gsqlparser.stmt.teradata.utilities.TeradataUtilityType;
import gudusoft.gsqlparser.util.TSnowflakeParameterChecker;


import java.io.*;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.*;

import static gudusoft.gsqlparser.ESqlStatementType.*;



/**
 * This is the first class people start to use this SQL parser library.
 * This class includes a lexer and a parser. The lexer is used to tokenize the input SQL, turn the input SQL text
 * into a list of source tokens. The parser use this list source token as input, using the grammar rule of the specified
 * database vendor to check the syntax of the input SQL and build the parse tree of this SQL if there is no syntax error
 * in the input SQL.
 * <p></p>
 * Creating a SQL parser by specifing {@link gudusoft.gsqlparser.EDbObjectType a database vendor},
 * then set SQL script text via {@link #setSqltext} method or reading the input SQL from a file via
 * {@link #setSqlfilename} method.
 * <p></p>
 * After that, call one of the following methods to achieve what you need:
 * <ul>
 *  <li>{@link #tokenizeSqltext}, turns the input SQL into a sequence of token which is the
 *  basic lexis element of SQL syntax. Token is categorized as keyword, identifier,
 *  number, operator, whitespace and other types. All source tokens can be fetched
 *  via the {@link #getSourcetokenlist()} method</li>
 *
 *  <li>{@link #getrawsqlstatements}, separates the SQL statements in the input SQL script without
 *  doing syntax check, use the {@link #getSqlstatements()} method to get a list of SQL statements
 *  which is the sub-class of {@link TCustomSqlStatement}, get SQL statement type
 *  via the {@link TCustomSqlStatement#sqlstatementtype} field, and string representation of
 *  each SQL statement via the {@link TCustomSqlStatement#toString} method. All source tokens in this SQL statement
 *  is available by using {@link TCustomSqlStatement#sourcetokenlist} filed.
 *  Since no parse tree is built by calling this method, no further detailed information about the SQL statement is available.
 *  </li>
 *
 *  <li>{@link #parse}, Check syntax of the input SQL, doing some kind of semantic analysis without connecting to
 *  a real database.
 *  This method will do a in-depth analysis of the input SQL such as building the link between table and columns.
 *  The parse tree of the input SQL is available after calling this method.
 *  </li>
 *  </ul>
 *
 *  The parser checks the syntax of those SQL statements one by one. If syntax error is found in a SQL statement,
 *  an error will be logged, no parse tree will be built for this SQL statement,
 *  the error message can be fetched using the {@link #getErrormessage()} method.
 *  <p></p>
 *  The syntax error in one SQL statement doesn't prevent the parser continue to check the syntax of the next SQL statement.
 *  After checking syntax of all SQL statements, use the {@link #getErrorCount()} method to get the total number of errors.
 *  <p></p>
 *  A syntax error in a SQL stored procedure will cease this parser to check syntax of the rest SQL statements
 *  in this stored procedure.
 *
 * <p>Format SQL script can be done after calling {@link #parse()}.
 * <code>
 *
 *      int ret = sqlparser.parse();
 *       if (ret == 0){
 *           GFmtOpt option = GFmtOptFactory.newInstance();
 *           String result = FormatterFactory.pp(sqlparser, option);
 *           System.out.println(result);
 *       }else{
 *           System.out.println(sqlparser.getErrormessage());
 *       }
 *
 * </code>
 *
 * <p> After paring SQL script, all parse tree nodes are available for use, some of use cases
 * are:
 * <ul>
 *     <li>Table/column impact analysis</li>
 *     <li>SQL rewriting</li>
 *     <li>SQL translate between different databases</li>
 *     <li>SQL migration analysis</li>
 *     <li>Help to anti SQL injection</li>
 *     <li><a href="http://support.sqlparser.com/">More use cases</a></li>
 *     </ul>
 *
 * <p>Typically, SQL parse tree nodes generated by this SQL Parser were closely related to SQL
 * elements defined in database vendor's SQL reference book. here is a brief summary of some
 * most used SQL elements and corresponding classes defined in this SQL parser.
 * <ul>
 *     <li>SQL identifier: {@link gudusoft.gsqlparser.nodes.TObjectName}</li>
 *     <li>SQL literal: {@link gudusoft.gsqlparser.nodes.TConstant}</li>
 *     <li>SQL datatype: {@link gudusoft.gsqlparser.nodes.TTypeName}</li>
 *     <li>SQL function: {@link gudusoft.gsqlparser.nodes.TFunctionCall}</li>
 *     <li>SQL constraint: {@link gudusoft.gsqlparser.nodes.TConstraint}</li>
 *     <li>SQL expression/condition: {@link gudusoft.gsqlparser.nodes.TExpression}</li>
 *     <li>SQL select list item: {@link gudusoft.gsqlparser.nodes.TResultColumn}</li>
 *      <li>More: {@link gudusoft.gsqlparser.nodes}</li>
 * </ul>
 *
 * <p> Some major SQL statements:
 * <ul>
 *      <li>Select: {@link gudusoft.gsqlparser.stmt.TSelectSqlStatement}</li>
 *      <li>Delete: {@link gudusoft.gsqlparser.stmt.TDeleteSqlStatement}</li>
 *      <li>Insert: {@link gudusoft.gsqlparser.stmt.TInsertSqlStatement}</li>
 *      <li>Update: {@link gudusoft.gsqlparser.stmt.TUpdateSqlStatement}</li>
 *      <li>Create table: {@link gudusoft.gsqlparser.stmt.TCreateTableSqlStatement}</li>
 *      <li>More: {@link   gudusoft.gsqlparser.stmt}</li>
 * </ul>
 *
 * <p>Stored procedure</p>
 * <ul>
 *     <li>Create function: {@link gudusoft.gsqlparser.stmt.db2.TDb2CreateFunction },
 *          {@link gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.mysql.TMySQLCreateFunction},
 *          {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateFunction}</li>
 *     <li>Create procedure: {@link gudusoft.gsqlparser.stmt.db2.TDb2CreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.mysql.TMySQLCreateProcedure},
 *          {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateProcedure}</li>
 *     <li>Create trigger: {@link gudusoft.gsqlparser.stmt.TCreateTriggerStmt},
 *          {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateTrigger}</li>
 *     <li>Create package: {@link gudusoft.gsqlparser.stmt.oracle.TPlsqlCreatePackage}</li>
 * </ul>
 *
 * For all available SQL parse tree node classes, please check the API reference.
 *
 *
 */
public class TGSqlParser {




    public void setSqlCharset(String sqlCharset) {
        this.sqlCharset = sqlCharset;
    }

    public String getSqlCharset() {
        return sqlCharset;
    }

    private String sqlCharset = null;

    public static EDbVendor currentDBVendor = EDbVendor.dbvoracle;


    /**
     * Turn the string name of database to dbvendor
     * <ul>
     *     <li>access: EDbVendor.dbvccess</li>
     *     <li>ansi: EDbVendor.dbvansi</li>
     *     <li>bigquery: EDbVendor.dbvbigquery</li>
     *     <li>couchbase: EDbVendor.dbvcouchbase</li>
     *     <li>dax: EDbVendor.dbvdax</li>
     *     <li>db2: EDbVendor.dbvdb2</li>
     *     <li>firebird: EDbVendor.dbvfirebird</li>
     *     <li>generic: EDbVendor.dbvgeneric</li>
     *     <li>greenplum: EDbVendor.dbvgreenplum</li>
     *     <li>hana: EDbVendor.dbvhana</li>
     *     <li>hive: EDbVendor.dbvhive</li>
     *     <li>impala: EDbVendor.dbvimpala</li>
     *     <li>informix: EDbVendor.dbvinformix</li>
     *     <li>mdx: EDbVendor.dbvmdx</li>
     *     <li>mssql or sqlserver: EDbVendor.dbvmssql</li>
     *     <li>mysql: EDbVendor.dbvmysql</li>
     *     <li>netezza: EDbVendor.dbvnetezza</li>
     *     <li>odbc: EDbVendor.dbvodbc</li>
     *     <li>openedge: EDbVendor.dbvopenedge</li>
     *     <li>oracle: EDbVendor.dbvoracle</li>
     *     <li>postgresql or postgres: EDbVendor.dbvpostgresql</li>
     *     <li>redshift: EDbVendor.dbvredshift</li>
     *     <li>snowflake: EDbVendor.dbvsnowflake</li>
     *     <li>sybase: EDbVendor.dbvsybase</li>
     *     <li>teradata: EDbVendor.dbvteradata</li>
     *     <li>vertica: EDbVendor.dbvvertica</li>
     * </ul>
     * @param dbVendorName
     * @return dbvendor
     */
    public static EDbVendor getDBVendorByName(String dbVendorName){
        return  EDbVendor.valueOfWithDefault(dbVendorName);
    }

//    public void  teradataCmds(){
//       // int cnt = 0;
//        //((TLexerTeradata)getFlexer()).
////         for(int i=0;i<TLexerTeradata.bteqCmdList.size();i++){
////             for(int j=0;j<TLexerTeradata.multiLoadCmdList.size();j++){
////                 if (TLexerTeradata.bteqCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.multiLoadCmdList.get(j).toString())){
////                   System.out.println("multiLoad: "+TLexerTeradata.bteqCmdList.get(i).toString());
////                 }
////             }
////             for(int j=0;j<TLexerTeradata.fastExportCmdList.size();j++){
////                 if (TLexerTeradata.bteqCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.fastExportCmdList.get(j).toString())){
////                     System.out.println("fastExport: "+TLexerTeradata.bteqCmdList.get(i).toString());
////                 }
////             }
////             for(int j=0;j<TLexerTeradata.fastLoadCmdList.size();j++){
////                 if (TLexerTeradata.bteqCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.fastLoadCmdList.get(j).toString())){
////                     System.out.println("FastLoad: "+TLexerTeradata.bteqCmdList.get(i).toString());
////                 }
////             }
////         } //bteqCmdList
//
////        for(int i=0;i<TLexerTeradata.fastLoadCmdList.size();i++){
////            for(int j=0;j<TLexerTeradata.multiLoadCmdList.size();j++){
////                if (TLexerTeradata.fastLoadCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.multiLoadCmdList.get(j).toString())){
////                    System.out.println("multiLoad: "+TLexerTeradata.fastLoadCmdList.get(i).toString());
////                }
////            }
////            for(int j=0;j<TLexerTeradata.fastExportCmdList.size();j++){
////                if (TLexerTeradata.fastLoadCmdList.get(i).toString().equalsIgnoreCase(TLexerTeradata.fastExportCmdList.get(j).toString())){
////                    System.out.println("fastExport: "+TLexerTeradata.fastLoadCmdList.get(i).toString());
////                }
////            }
////
////        }
//
//    }

    private Stack<TFrame> frameStack = null;

    public Stack<TFrame> getFrameStack(){
        if (frameStack == null){
            frameStack = new Stack<TFrame>();
        }

        return  frameStack;
    }

    public void setFrameStack(Stack<TFrame> frameStack) {
        this.frameStack = frameStack;
    }

    void closeFileStream(){
        if (streamFromSqlFile != null) {
            try {
                streamFromSqlFile.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    FileInputStream streamFromSqlFile = null;
    InputStreamReader sqlStreamReader = null;
    /**
     * A sequence of source tokens created by the lexer after tokenize the input SQL
     *
     * @return a sequence of source tokens
     */
    public TSourceTokenList getSourcetokenlist() {
        return sourcetokenlist;
    }

    /**
     * A list of SQL statements created by the parser.
     * If this list is created after calling the {@link #getrawsqlstatements} method, the syntax of each SQL statement
     * is not checked and the parse tree of each statement is not created. If the {@link #parse} method is called to build
     * this SQL statement list, then every thing is ready.
     *
     * @return a list of SQL statement
     */
    public TStatementList getSqlstatements() {
        return sqlstatements;
    }

    enum stored_procedure_status {start,is_as,body,bodyend,end, cursor_declare};
    enum stored_procedure_type {function,procedure,package_spec,package_body, block_with_begin,block_with_declare,
        create_trigger,create_library,cursor_in_package_spec,others};

    static final int stored_procedure_nested_level = 1024;

    /**
    ** The input SQL Text.
    ** If {@link #sqlfilename} is specified, then this field will be ignored.
    */
    public String sqltext;

    /**
     * set the input SQL text, If {@link #sqlfilename} is specified before this method, the parser will using
     * the SQL text in this field instead of read SQL from {@link #sqlfilename}.
     *
     * @param sqltext the input SQL text
     */
    public void setSqltext(String sqltext) {
        this.sqltext = sqltext;
        this.sqlfilename = "";
        this.sqlInputStream = null;
    }

    /**
     *  The SQL text that being processed.
     *
     * @return the SQL text that being processed
     */
    public String getSqltext() {
        return sqltext;
    }


    /**
    ** The input SQL will be read from this file
     *
    ** If field is specified, then {@link #sqltext} will be ignored.
     * This must be the full path to the file, relative path doesn't work.
    */
    public String sqlfilename;


    /**
     * set the filename from which the input SQL will be read.
     *
     * @param sqlfilename the SQL file name from which the input SQL will be read
     */
    public void setSqlfilename(String sqlfilename) {
        this.sqlfilename = sqlfilename;
        this.sqltext = "";
        this.sqlInputStream = null;
    }

    /**
     * The input SQL filename. This parser can process the unicode encoded SQL file.
     *
     * @return the input SQL filename
     */
    public String getSqlfilename() {
        return sqlfilename;
    }

    /**
     * set the InputStream from which SQL will be read.
     * If this method is called, {@link #sqlfilename} and {@link #sqltext} will be ignored.
     *
     * @param sqlInputStream the InputStream from which SQL will be read
     */
    public void setSqlInputStream(InputStream sqlInputStream) {
        if (sqlInputStream instanceof BufferedInputStream){
            this.sqlInputStream = (BufferedInputStream)sqlInputStream;
        }else{
            this.sqlInputStream = new BufferedInputStream(sqlInputStream);
        }

        this.sqlfilename = "";
        this.sqltext = "";
    }

    private BufferedInputStream  sqlInputStream;

    /**
     *  the InputStream from which SQL will be read
     * @return the InputStream from which SQL will be read
     */
    public InputStream getSqlInputStream() {
        return sqlInputStream;
    }

    /**
    ** Tokens generated by lexer from the input SQL script.
     * Tokens are always available even if there are syntax errors in input the SQL script.
    */
    public TSourceTokenList sourcetokenlist;

    /**
    ** SQL statements generated by this parser from the input SQL script.
     * statements are always available even if there are syntax errors in input SQL script.
     * if there is no syntax error in a statement, you can access the parse tree of the statement to fetch more information
     * such as tables, columns, etc.
    */
    public TStatementList sqlstatements;


    /**
     * The array of syntax error generated by the parser during checking the syntax of the input SQL,
     * element of this list is type of {@link TSyntaxError}
     *
     * @return the array of errors
     */
    public ArrayList <TSyntaxError> getSyntaxErrors() {
        return syntaxErrors;
    }

    private ArrayList <TSyntaxError> syntaxErrors;

//    public ArrayList<TSyntaxError> getSyntaxHints() {
//        return syntaxHints;
//    }

    //private ArrayList <TSyntaxError> syntaxHints;

    /**
     * The database vendor specified when creating this parser.
     * The grammar rule of this database vendor will be used to validate the syntax of the input SQL.
     *
     * @return the database vendor
     */
    public EDbVendor getDbVendor() {
        return dbVendor;
    }

    /**
     * @deprecated As of v1.4.3.4
     * enable GSP to parse the rest of sql statements inside stored procedure
     * when a SQL statement in the stored procedure cannot be parsed
     *
     * <p>Available to parse sybase stored procedure currently.
     *
     * @param enablePartialParsing set true to enable this partial parsing, default is false
     */
    public void setEnablePartialParsing(boolean enablePartialParsing) {
        this.enablePartialParsing = enablePartialParsing;
    }

    /**
     * enable GSP to parse the rest of sql statements inside stored procedure
     * when a SQL statement in the stored procedure cannot be parsed
     *
     * <p>Available to parse sybase stored procedure currently.
     *
     * <p> default is false;
     *
     *  @deprecated As of v1.4.3.4
     */
    private boolean isEnablePartialParsing() {

        return enablePartialParsing;
    }

    private boolean enablePartialParsing = false;

    private static String userName;
    private static String machineId = null;
    private static String licenseKey;
    private static String licenseType;
    private static boolean licenseOK = false;
    private static String licenseMessage;

    /**
     * Not used.
     *
     * @return the user name
     */
    public static String getUserName() {
        return userName;
    }

    /**
     * Not used.
     *
     * @return the machine id
     */
    public static String getMachineId() {

        return machineId;
    }

    /**
     * Not used.
     *
     * @return the license message
     */
    public static String getLicenseMessage() {
        return licenseMessage;
    }


    static {
        licenseOK = validateLicense();
    }

    /**
     * Not used.
     *
     * @return trial license or developer license or distribution license
     */
    public static String getLicenseType() {
        return licenseType;
    }

    private EDbVendor dbVendor;
    private String errormessage;

    /**
     * The lexer which is used to tokenize the input SQL.
     *
     * @return the lexer
     */
    public TCustomLexer getFlexer() {
        return flexer;
    }

    private TCustomLexer flexer;
    TCustomParser fparser,fplsqlparser;
    BufferedReader finputstream = null; //used by lexer
    TCustomSqlStatement gcurrentsqlstatement,nextStmt;
    TSqlCmds sqlcmds;

    HashMap sqlpluskeywordList;

    char delimiterchar;
    String defaultDelimiterStr;

    private ISQLStatementHandle sqlStatementHandle = null;

    public void setSqlStatementHandle(ISQLStatementHandle sqlStatementHandle) {
        this.sqlStatementHandle = sqlStatementHandle;
    }

    private ITokenHandle tokenHandle = null;

    /**
     * Set an event handler which will be fired when a new source token is created by the lexer during tokenize the
     * input SQL.
     *
     * @param tokenHandle the event handler to process the new created source token
     */
    public void setTokenHandle(ITokenHandle tokenHandle) {
        this.tokenHandle = tokenHandle;
    }

    private ITokenListHandle tokenListHandle = null;

    public void setTokenListHandle(ITokenListHandle tokenListHandle) {
        this.tokenListHandle = tokenListHandle;
    }

    private IMetaDatabase metaDatabase = null;

    /**
     * @deprecated As of v2.0.3.1, please use {@link #getSqlEnv()} instead
     *
     *  set an instance of a class which implement the interface: {@link IMetaDatabase}.
     *  The parser will call {@link IMetaDatabase#checkColumn} method when it needs to know
     *  whether a column is belonged to a table.
     *  <p></p>
     *  The class that implements the interface: {@link IMetaDatabase} usually fetch the metadata from the database
     *  by connecting to a database instance.
     *  <p></p>
     *  If the class is not provided, the parser has to guess the relationship between a un-qualified column and table
     *  in the input SQL which may lead to a un-determined result between the column and table.
     *
     * @param metaDatabase a new instance of the class which implements the {@link IMetaDatabase} interface
     * @see IMetaDatabase
     */
    public void setMetaDatabase(IMetaDatabase metaDatabase) {
        this.metaDatabase = metaDatabase;
    }

    /**
     * @deprecated As of v2.0.3.1, please use {@link #getSqlEnv()} instead
     *
     * a new instance of the class which implements the {@link IMetaDatabase} interface
     *
     * @return a new instance of the class which implements the {@link IMetaDatabase} interface
     * @see #setMetaDatabase
     */
    public IMetaDatabase getMetaDatabase() {

        return metaDatabase;
    }

    /**
     * Not used.
     *
     */
    public void freeParseTable() {
        flexer.yystack = null;
        flexer.yytextbuf = null;
        flexer.buf = null;

//        TLexerOracle.yyk = null;
//        TLexerOracle.yykl = null;
//        TLexerOracle.yykh = null;
//        TLexerOracle.yym = null;
//        TLexerOracle.yyml = null;
//        TLexerOracle.yymh = null;
//        TLexerOracle.yyt = null;
//        TLexerOracle.yytl = null;
//        TLexerOracle.yyth = null;
//        TParserOracleSql.yyah = null;
//        TParserOracleSql.yyal = null;
//        TParserOracleSql.yygh = null;
//        TParserOracleSql.yygl = null;
//        TParserOracleSql.yyd = null;
//        TParserOracleSql.yya_sym= null;
//        TParserOracleSql.yya_act= null;
//        TParserOracleSql.yyr_len= null;
//        TParserOracleSql.yyr_sym= null;
//        TParserOracleSql.yyg_sym= null;
//        TParserOracleSql.yyg_act= null;
//
//        TParserOraclePLSql.yyah = null;
//        TParserOraclePLSql.yyal = null;
//        TParserOraclePLSql.yygh = null;
//        TParserOraclePLSql.yygl = null;
//        TParserOraclePLSql.yyd = null;
//        TParserOraclePLSql.yya_sym= null;
//        TParserOraclePLSql.yya_act= null;
//        TParserOraclePLSql.yyr_len= null;
//        TParserOraclePLSql.yyr_sym= null;
//        TParserOraclePLSql.yyg_sym= null;
//        TParserOraclePLSql.yyg_act= null;
//
//        TLexerMssql.yyk = null;
//        TLexerMssql.yykl = null;
//        TLexerMssql.yykh = null;
//        TLexerMssql.yym = null;
//        TLexerMssql.yyml = null;
//        TLexerMssql.yymh = null;
//        TLexerMssql.yyt = null;
//        TLexerMssql.yytl = null;
//        TLexerMssql.yyth = null;
//        TParserMssqlSql.yyah = null;
//        TParserMssqlSql.yyal = null;
//        TParserMssqlSql.yygh = null;
//        TParserMssqlSql.yygl = null;
//        TParserMssqlSql.yyd = null;
//        TParserMssqlSql.yya_sym= null;
//        TParserMssqlSql.yya_act= null;
//        TParserMssqlSql.yyr_len= null;
//        TParserMssqlSql.yyr_sym= null;
//        TParserMssqlSql.yyg_sym= null;
//        TParserMssqlSql.yyg_act= null;
//
//        TLexerMysql.yyk = null;
//        TLexerMysql.yykl = null;
//        TLexerMysql.yykh = null;
//        TLexerMysql.yym = null;
//        TLexerMysql.yyml = null;
//        TLexerMysql.yymh = null;
//        TLexerMysql.yyt = null;
//        TLexerMysql.yytl = null;
//        TLexerMysql.yyth = null;
//        TParserMysqlSql.yyah = null;
//        TParserMysqlSql.yyal = null;
//        TParserMysqlSql.yygh = null;
//        TParserMysqlSql.yygl = null;
//        TParserMysqlSql.yyd = null;
//        TParserMysqlSql.yya_sym= null;
//        TParserMysqlSql.yya_act= null;
//        TParserMysqlSql.yyr_len= null;
//        TParserMysqlSql.yyr_sym= null;
//        TParserMysqlSql.yyg_sym= null;
//        TParserMysqlSql.yyg_act= null;
    }

    /**
     *  Class constructor, create a new instance of the parser by setting the database vendor
     *
     * @param pdbvendor the database vendor whose grammar rule will be used to validate the syntax of the input SQL
     */
    public TGSqlParser(EDbVendor pdbvendor) {
        dbVendor = pdbvendor;
        sqltext = "";
        sqlfilename = "";

        delimiterchar = ';';
        defaultDelimiterStr = ";";
        switch(pdbvendor){
            case dbvmssql :
            case dbvaccess:
            case dbvazuresql:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sqlserver_edition||TBaseType.azuresql_edition){
                    flexer = new TLexerMssql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMssqlSql(null);
                    fparser.lexer = flexer;
                }
                if (pdbvendor == EDbVendor.dbvazuresql){
                    dbVendor = EDbVendor.dbvmssql;
                }
                break;
            }
            case dbvsybase:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sybase_edition){
                    flexer = new TLexerSybase();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSybase(null);
                    fparser.lexer = flexer;
                }
                enablePartialParsing = false;
                break;
            }
            case dbvinformix:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.informix_edition){
                    flexer = new TLexerInformix();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserInformix(null);
                    fparser.lexer = flexer;
                }
                enablePartialParsing = false;
                break;
            }
            case dbvoracle :{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.oracle_edition){
                    delimiterchar = '/';
                    flexer = new TLexerOracle();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserOracleSql(null);
                    fplsqlparser = new TParserOraclePLSql(null);
                    fparser.lexer = flexer;
                    fplsqlparser.lexer = flexer;
                    fplsqlparser.getNf().setGsqlParser(this);
                }
                break;
            }
//            case dbvaccess :{
//                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sqlserver_edition){
//                    flexer = new TLexerAccess();
//                    flexer.delimiterchar = delimiterchar;
//                    flexer.defaultDelimiterStr = defaultDelimiterStr;
//                    fparser = new TParserAccessSql(null);
//                    fparser.lexer = flexer;
//                }
//                break;
//            }
            case dbvdb2 :{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.db2_edition){
                    delimiterchar = '@';
                    flexer = new TLexerDb2();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserDb2Sql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvmysql :{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.mysql_edition){
                    delimiterchar = '$';
                    defaultDelimiterStr = "$";
                    flexer = new TLexerMysql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMysqlSql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvteradata :{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.teradata_edition){
                    delimiterchar = '/';
                    flexer = new TLexerTeradata();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserTeradata(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvpostgresql:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.postgresql_edition){
                    delimiterchar = '/';
                    flexer = new TLexerPostgresql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserPostgresql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvredshift:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.redshift_edition){
                    delimiterchar = '/';
                    flexer = new TLexerRedshift();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserRedshift(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvgreenplum:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.greenplum_edition){
                    delimiterchar = '/';
                    flexer = new TLexerGreenplum();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserGreenplum(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvmdx:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.mdx_edition){
                    delimiterchar = ';';
                    flexer = new TLexerMdx();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMdx(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvnetezza:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.netezza_edition){
                    delimiterchar = ';';
                    flexer = new TLexerNetezza();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserNetezza(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvhive:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.hive_edition){
                    delimiterchar = ';';
                    flexer = new TLexerHive();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserHive(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvimpala:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.impala_edition){
                    delimiterchar = ';';
                    flexer = new TLexerImpala();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserImpala(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvhana:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.hana_edition){
                    delimiterchar = ';';
                    flexer = new TLexerHana();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserHana(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvdax:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.dax_edition){
                    delimiterchar = ';';
                    flexer = new TLexerDax();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserDax(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvodbc:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.odbc_edition){
                    delimiterchar = ';';
                    flexer = new TLexerOdbc();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserOdbc(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvvertica:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.vertica_edition){
                    delimiterchar = ';';
                    flexer = new TLexerVertica();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserVertica(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvopenedge:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.openedge_edition){
                    delimiterchar = ';';
                    flexer = new TLexerOpenedge();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserOpenedge(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvcouchbase:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.couchbase_edition){
                    delimiterchar = ';';
                    flexer = new TLexerCouchbase();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserCouchbase(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvsnowflake:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.snowflake_edition){
                    delimiterchar = ';';
                    flexer = new TLexerSnowflake();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSnowflake(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvbigquery:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.bigquery_edition){
                    delimiterchar = ';';
                    flexer = new TLexerBigquery();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserBigquery(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvansi:
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.ansi_edition){
                    delimiterchar = ';';
                    flexer = new TLexerAnsi();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserAnsi(null);
                    fparser.lexer = flexer;
                }
                break;
            case dbvgeneric: {
                if (TBaseType.enterprise_edition || (!TBaseType.full_edition) || TBaseType.generic_edition) {
                    flexer = new TLexerMssql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMssqlSql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvsoql:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.soql_edition){
                    delimiterchar = ';';
                    flexer = new TLexerSoql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSoql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvsparksql:{
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sparksql_edition){
                    delimiterchar = ';';
                    flexer = new TLexerSparksql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserSparksql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvathena:
                {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.athena_edition){
                    delimiterchar = ';';
                    flexer = new TLexerathena();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserAthena(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvpresto:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.presto_edition){
                    delimiterchar = ';';
                    flexer = new TLexerPresto();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserPresto(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvtrino:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.trino_edition){
                    delimiterchar = ';';
                    flexer = new TLexerPresto();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserPresto(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvdatabricks:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.databricks_edition){
                    delimiterchar = ';';
                    flexer = new TLexerDatabricks();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserDatabricks(null);
                    fparser.lexer = flexer;
                }
                break;
            }
            case dbvgaussdb:
            {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.gaussdb_edition){
                    delimiterchar = ';';
                    flexer = new TLexerGaussDB();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserGaussDB(null);
                    fparser.lexer = flexer;
                   // fplsqlparser = new TParserOraclePLSql(null);
                   // fplsqlparser.lexer = new TLexerOracle();// must use oracle lexer, not gauss lexer
                }
                break;
            }
            default:  {
                if (TBaseType.enterprise_edition ||(!TBaseType.full_edition) ||TBaseType.sqlserver_edition){
                    flexer = new TLexerMssql();
                    flexer.delimiterchar = delimiterchar;
                    flexer.defaultDelimiterStr = defaultDelimiterStr;
                    fparser = new TParserMssqlSql(null);
                    fparser.lexer = flexer;
                }
                break;
            }
        }

        fparser.getNf().setGsqlParser(this);

        sourcetokenlist = new TSourceTokenList();
        sourcetokenlist.setGsqlparser(this);
        sqlstatements = new TStatementList();
        sqlcmds = new TSqlCmds() ;
        sqlpluskeywordList = new HashMap();
        syntaxErrors = new ArrayList();
        //syntaxHints = new ArrayList();


        errormessage = "";

        if (TBaseType.license_expired_check){
            if (!check_license_time()) {
                flexer = null;
                fparser = null;
               // System.out.println("license is expired");
            }
        }

        TGSqlParser.currentDBVendor = dbVendor;
    }

    TContext globalContext;

//     public TExpression parseExpression(String expr){
//        sqltext = "select"+TBaseType.newline+expr+TBaseType.newline+"from t";
//        int iRet = doparse();
//        if (iRet!= 0) {
//            for (int i= 0; i< syntaxErrors.size(); i++)
//            {
//                TSyntaxError t = (TSyntaxError) syntaxErrors.get(i);
//                t.lineNo--;
//            }
//
//            return null;
//        }
//        return ((TSelectSqlStatement)sqlstatements.get(0)).getResultColumnList().getResultColumn(0).getExpr();
//    }

    /**
     * Create a select statement object from the parameter: subquery
     *
     * @param subquery a plain text select statement which need to be converted to a {@link gudusoft.gsqlparser.stmt.TSelectSqlStatement} object
     * @return a select statement object, return null if the input select string is syntax invalid.
     */
    public TSelectSqlStatement parseSubquery(String subquery){
        return parseSubquery(this.dbVendor,subquery);
    }

    public static TSelectSqlStatement parseSubquery(EDbVendor dbVendor, String subquery){
//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = subquery;
//        int iRet = localParser.doparse();
//        if (iRet != 0) {
//            return null;
//        }

        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,subquery);
        if (statements.size() == 0) return null;

        return (TSelectSqlStatement)statements.get(0);
    }

    /**
     * Create an expression object from the parameter: expr
     *
     * @param expr a plain text expression which will be converted to {@link gudusoft.gsqlparser.nodes.TExpression} object
     * @return an expression object, return null if the input expression string is not syntax valid.
     */
    public TExpression parseExpression(String expr){
        return parseExpression(this.dbVendor,expr);
    }
    public static TExpression parseExpression(EDbVendor dbVendor, String expr){
        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,"select 1 from t where "+TBaseType.newline+expr);
        if (statements.size() == 0) return null;

//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select 1 from t where "+TBaseType.newline+expr;
//        int iRet = localParser.doparse();
//        if (iRet != 0) {
//            return null;
//        }

        return ((TSelectSqlStatement)statements.get(0)).getWhereClause().getCondition();
    }

    /**
     *  Create a function object from the parameter: newFunction
     *
     * @param newFunction a plain text function which will be converted to {@link gudusoft.gsqlparser.nodes.TFunctionCall} object
     * @return a function object, or return null if the input string is not a valid function call.
     */
    public TFunctionCall parseFunctionCall(String newFunction){
        return parseFunctionCall(this.dbVendor,newFunction);
    }

    public static TFunctionCall parseFunctionCall(EDbVendor dbVendor, String newFunction){
        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,"select"+TBaseType.newline+newFunction+TBaseType.newline+"from t");
        if (statements.size() == 0) return null;


//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select"+TBaseType.newline+newFunction+TBaseType.newline+"from t";
//        int iRet = localParser.doparse();
//        if (iRet!= 0) {
//            return null;
//        }

        return statements.get(0).getResultColumnList().getResultColumn(0).getExpr().getFunctionCall();
    }

    /**
     * Create a database objectName from the parameter: newObjectName
     *
     * @param newObjectName a plain text objectName which will be converted to {@link gudusoft.gsqlparser.nodes.TObjectName} object
     * @return a database objectName object, return null is the input string is not a valid objectName.
     */
    public TObjectName parseObjectName(String newObjectName){
        //return parseObjectName(this.dbVendor,newObjectName);
        return TObjectName.createObjectName(this.dbVendor,EDbObjectType.column,newObjectName);
    }



    /**
     *
     * @param dbVendor
     * @param newObjectName
     * @return
     */
    public static TObjectName parseObjectName(EDbVendor dbVendor, String newObjectName){

        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,"select"+TBaseType.newline+newObjectName+TBaseType.newline+"from t");
        if (statements.size() == 0) return null;
        TExpression e = statements.get(0).getResultColumnList().getResultColumn(0).getExpr();

//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select"+TBaseType.newline+newObjectName+TBaseType.newline+"from t";
//        int iRet = localParser.doparse();
//        if (iRet!= 0) {
//            return null;
//        }
//        TExpression e = ((TSelectSqlStatement)localParser.sqlstatements.get(0)).getResultColumnList().getResultColumn(0).getExpr();

        TObjectName lcResult = null;
        switch (e.getExpressionType()){
            case simple_object_name_t:
                lcResult = e.getObjectOperand();
                break;
            case simple_constant_t:
                lcResult = new TObjectName();
                lcResult.init(e.getConstantOperand().getValueToken());
                break;
            default:
                break;
        }

        return lcResult;
    }

    /**
     * Create an constant object from the parameter: newConstant
     *
     * @param newConstant a plian text constant which will be converted to {@link gudusoft.gsqlparser.nodes.TConstant} object
     * @return new constant object, returns null if the input is not a valid constant string.
     */
    public TConstant parseConstant(String newConstant){
        return parseConstant(this.dbVendor,newConstant);
    }

    public static TConstant parseConstant(EDbVendor dbVendor, String newConstant){
//        TGSqlParser localParser = new TGSqlParser(dbVendor);
//        localParser.sqltext = "select"+TBaseType.newline+newConstant+TBaseType.newline+"from t";
//        int iRet = localParser.doparse();
//        if (iRet!= 0) {
//            return null;
//        }

        TSingletonParser singletonParser = TSingletonParser.getInstance();
        TStatementList statements = singletonParser.getStmts(dbVendor,"select"+TBaseType.newline+newConstant+TBaseType.newline+"from t");
        if (statements.size() == 0) return null;

        return statements.get(0).getResultColumnList().getResultColumn(0).getExpr().getConstantOperand();
    }


    /**
     * The total number of syntax errors founded in the input SQL script.
     * <br>Only the first syntax error in a SQL statement is counted if there are more than one syntax errors in
     * a single SQL statement.
     * <br> In a SQL script, only the first syntax error in each SQL statement is counted.
     *
     * @return The total number of syntax errors founded in the input SQL script. Returns 0 if no syntax error is founded.
     */
    public int getErrorCount(){
        return   syntaxErrors.size();
    }

    /**
     * The text of error message generated by iterating all items in {@link #getSyntaxErrors}.
     * User may generate error message in their own format by iterating all items in {@link #getSyntaxErrors}.
     *
     * @return error message
     */
    public String getErrormessage(){

        String s="",hint="Syntax error";
        TSyntaxError t;
        for (int i= 0; i< syntaxErrors.size(); i++)
        {
            t = (TSyntaxError) syntaxErrors.get(i);
            if (t.hint.length() > 0) hint = t.hint;
            s= s+hint+"("+t.errorno+") near: "+t.tokentext;
            s=s+"("+t.lineNo;
            s=s+","+t.columnNo +", token code:"+t.tokencode+")";
                //s=s+" expected tokentext:"+t.hint;

           // break;//get only one message, remove this one and uncomment next line to get all error messages
            if (i !=  syntaxErrors.size() - 1)
              s = s +TBaseType.linebreak;
        }

        if (errormessage.length() > 0){
            s = errormessage+TBaseType.linebreak+s; 
        }
        return s;

    }

    /**
     * check syntax of the input SQL. This method works exactly the same as {@link #parse} method.
     *
     * @return   0 means parse SQL script successfully, otherwise, use {@link #getErrorCount}, {@link #getErrormessage}
     * to get detailed error information.
     * @see #parse
     */
    public int checkSyntax(){
        return doparse();
    }

    /**
     *   Check syntax of the input SQL, doing some kind of semantic analysis without connecting to a real database.
     *   <p></p>
     *  This method will do a in-depth analysis of the input SQL such as building the link between table and columns.
     *  The parse tree of the input SQL is available after calling this method.
     *
     *  The parser checks the syntax of those SQL statements one by one. If syntax error is found in a SQL statement,
     *  an error will be logged, no parse tree will be built for this SQL statement,
     *  the error message can be fetched using the {@link #getErrormessage()} method.
     *  <p></p>
     *  The syntax error in one SQL statement doesn't prevent the parser continue to check the syntax of the next SQL statement.
     *  After checking syntax of all SQL statements, use the {@link #getErrorCount()} method to get the total number of errors.
     *  <p></p>
     *  A syntax error in a SQL stored procedure will cease this parser to check syntax of the rest SQL statements
     *  in this stored procedure.
     *
     * @return   0 means parse SQL script successfully, otherwise, use {@link #getErrorCount}, {@link #getErrormessage}
     * to get detailed error information.
     * @see #getSyntaxErrors
     */

    public int parse(){
        return doparse();
    }

    public int validate(){
        if (sqlstatements.size() == 0) return 0;

        TRelationValidator relationValidate = new TRelationValidator(globalContext);
        for(TCustomSqlStatement sqlStatement:sqlstatements){
            sqlStatement.acceptChildren(relationValidate);
        }
        return 0;
    }

    void setdelimiterchar(char ch){
        delimiterchar = ch;
        flexer.delimiterchar = ch;
    }
    char curdelimiterchar;
    String userDelimiterStr ="";

    boolean includesqlstatementtype(ESqlStatementType search, ESqlStatementType[] src){
        boolean ret = false;
        for(int i=0;i<src.length;i++){
            if (src[i] == search){
                ret = true;
                break;
            }
        }
        return ret;
    }

    // private int getfileEncodingType(FileInputStream inputStream){
    //     BufferedInputStream fr = new BufferedInputStream(inputStream,8);
    //     return getfileEncodingType(fr);
    // }

    private int getfileEncodingType(BufferedInputStream fr){
        int ret = 0; // default, 1: utf-16, 2: utf-32
       // BufferedInputStream fr = new BufferedInputStream(inputStream,8);
        try {
            byte[] bom = new byte[4];
            fr.mark(bom.length+1);

            fr.read(bom,0,bom.length);
            if ( ((bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE))
                    ||((bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF))
                    )
            {
                ret = 1;
                if ( ((bom[2] == (byte)0xFF) && (bom[3] == (byte)0xFE))
                        ||((bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF))
                        ){
                    ret = 2;
                }
            }else{
                if ((bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB)&& (bom[2] == (byte)0xBF)){
                    ret = 3; //UTF-8,EF BB BF
                }
            }
            fr.reset();
        } catch (FileNotFoundException e) {
            // e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
        } catch (IOException e) {
            //e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
        }
        return ret;
    }

    // private int getfileEncodingType(String fn){
    //     int ret = 0; // default, 1: utf-16, 2: utf-32
    //     try {
    //         FileInputStream fr =   new FileInputStream(fn);
    //         ret = getfileEncodingType(fr);
    //         fr.close();

    //     } catch (FileNotFoundException e) {
    //        // e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    //     } catch (IOException e) {
    //         //e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    //     }
    //     return ret;
    // }

    int readsql(){
        int ret  = 0;

        syntaxErrors.clear();
        //syntaxHints.clear();

        if (((!TBaseType.full_edition))||(!TBaseType.need_license_file)){
            if (!TBaseType.need_license_file){
                licenseType = TBaseType.license_type_dist;
                userName = "dist";
            }else {
                licenseType = TBaseType.license_type_trial;
                userName = TBaseType.license_trail_username;
            }
           // machineId = new HardwareBinder().getMachineIdString();
        }else {
            if (!licenseOK){
                errormessage = licenseMessage;
                return -1;
            }
        }

        try{
            if (finputstream != null) finputstream.close();
            if (flexer == null){
                ret = -1;
                errormessage = "requested database not supported:"+this.dbVendor.toString();
                return ret;
            }
            if (flexer.yyinput != null)    flexer.yyinput.close();

        }catch(IOException e){
            ret = -1;
            errormessage = "requested database not supported";
        }
        
        if (sqltext.length() > 0){
            finputstream = new BufferedReader(new StringReader(sqltext));
            if ((!TBaseType.full_edition) && (sqltext.length() > TBaseType.query_size_limitation)){
                errormessage = TBaseType.trail_version_query_message;
                ret = -1;
            }
        }else if (sqlfilename.length() > 0){
            try{

             streamFromSqlFile =   new FileInputStream(sqlfilename);
            // Buffer it for mark/reset support
            BufferedInputStream bufferedStream = new BufferedInputStream(streamFromSqlFile,8);             
            int encodingtype = getfileEncodingType(bufferedStream);
            streamFromSqlFile.getChannel().position(0);

           if(encodingtype == 1){
                sqlStreamReader = new InputStreamReader(streamFromSqlFile,"UTF-16");
           }else if(encodingtype == 2){
                sqlStreamReader = new InputStreamReader(streamFromSqlFile,"UTF-32");
           }else if(encodingtype == 3){
              // System.out.println("utf-8");
                sqlStreamReader = new InputStreamReader(streamFromSqlFile,"UTF-8");
           }
           else{
               if (sqlCharset == null){
                   sqlCharset = Charset.defaultCharset().name();
                   //System.out.println("Charset used: "+Charset.defaultCharset().name());
               }
                sqlStreamReader = new InputStreamReader(streamFromSqlFile, sqlCharset);
              // isr = new InputStreamReader(fr,"Cp737");
           }

            finputstream =  new BufferedReader(sqlStreamReader);
            if (encodingtype == 3){
                //EF BB BF was not stripped by the InputStreamReader, so we do it
                finputstream.skip(1);
            }

            if ((!TBaseType.full_edition)){
                File file = new File(sqlfilename);
                if (!file.exists() || !file.isFile()) {
                    ret = -1;
                    errormessage = "not a valid sql file.";
                }else{
                    if (file.length() > TBaseType.query_size_limitation){
                        errormessage = TBaseType.trail_version_file_message;
                        ret = -1;
                    }
                }
            }

            }catch(FileNotFoundException e){
                ret = -1;
                errormessage = e.toString();
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            } catch (IOException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }

        }else if(this.sqlInputStream != null){
            int encodingtype = getfileEncodingType(sqlInputStream);
            InputStream fr =   sqlInputStream;

            InputStreamReader isr = null;
            try{

                if(encodingtype == 1){
                    isr = new InputStreamReader(fr,"UTF-16");
                }else if(encodingtype == 2){
                    isr = new InputStreamReader(fr,"UTF-32");
                }else if(encodingtype == 3){
                    // System.out.println("utf-8");
                    isr = new InputStreamReader(fr,"UTF-8");
                }
                else{
                    if (sqlCharset == null){
                        sqlCharset = Charset.defaultCharset().name();
                    }

                    isr = new InputStreamReader(fr, sqlCharset);
                }

                finputstream =  new BufferedReader(isr);
                if (encodingtype == 3){
                    //EF BB BF was not stripped by the InputStreamReader, so we do it
                    finputstream.skip(1);
                }

            }catch(FileNotFoundException e){
                ret = -1;
                errormessage = e.toString();
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            } catch (IOException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }

            try {
                if ((!TBaseType.full_edition) && (sqlInputStream.available() > TBaseType.query_size_limitation)){
                    errormessage = TBaseType.trail_version_query_message;
                    ret = -1;
                }
            } catch (IOException e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }
        }

       if (finputstream == null) ret = -1;

       if (ret == 0)
       {
           flexer.yyinput =  finputstream;
           flexer.setSqlCharset(this.sqlCharset);
           flexer.reset();

       }

        sourcetokenlist.clear();
        sourcetokenlist.curpos = -1;

       flexer.resetTokenTable();


        return ret;
    }

    TSourceToken getanewsourcetoken(){
        TSourceToken pst = null,prevst;
        
        while (true) {
            pst = new TSourceToken("");
            if (flexer.yylexwrap(pst) == 0) { pst = null; break;}
            
            pst.setDbvendor(dbVendor);
            pst.tokenstatus = ETokenStatus.tsoriginal;
            if (pst.tokentype == ETokenType.ttreturn){
               pst.astext = towinlinebreak(pst.astext);
            }
            //combine space && linebreak after a linebreak into one
            if ( (pst.tokentype == ETokenType.ttwhitespace)
             && (sourcetokenlist.curpos >= 0) )
              {
                prevst =  sourcetokenlist.get(sourcetokenlist.curpos);
                if ( prevst.tokentype == ETokenType.ttreturn )
                  {
                    //can't discard  whitespace after linebreak, it will be used
                    // to judge whether / at the { of the line is a sqlplus cmd or not
                    // check isvalidplacefordivtosqlpluscmd for more
                    prevst.astext = prevst.astext+pst.astext;
                    //newst.free;
                    //newst = nil;
                    continue;
                  }
              }

            if ( (pst.tokentype == ETokenType.ttreturn)
             && (sourcetokenlist.curpos >= 0) )
              {
                prevst =  sourcetokenlist.get(sourcetokenlist.curpos);

                if ( prevst.tokentype == ETokenType.ttreturn )
                  {
                    prevst.astext = prevst.astext+pst.astext;
                    //newst.free;
                    //newst = nil;
                    continue;
                  }

                if ( prevst.tokentype == ETokenType.ttwhitespace )
                  {
                    //merge previous whitespace with this linebreak
                    //doesn't work for sqlplus cmd

                   // prevst.sourcecode = newst.sourcecode;
                   // prevst.tokentype = newst.tokentype;
                   // prevst.tokencode = newst.tokencode;
                   // newst.free;
                   // newst = nil;
                   // continue;

                   //prevst.prevsourcecode = prevst.astext;
                   //prevst.astext =  "";
                  }
              }

            break;

        }

        if (pst != null){
            pst.container = sourcetokenlist;
            sourcetokenlist.curpos = sourcetokenlist.curpos+1;
            pst.posinlist = sourcetokenlist.curpos;
            if (tokenHandle != null){
                tokenHandle.processToken(pst);
            }
        }

       // System.out.println(pst);
        flexer.setTokenTableValue(pst);
        return pst;
        
    }

  String towinlinebreak(String s){
      return s;
// todo not implemented yet     
  }

void checkconstarinttoken(TSourceToken lcprevtoken){
    TSourceTokenList lcStList = lcprevtoken.container;
    if (TBaseType.assigned(lcStList))
    {
         TSourceToken lcPPToken = lcStList.nextsolidtoken(lcprevtoken.posinlist,-2,false);
         if (TBaseType.assigned(lcPPToken))
        {

             if (lcPPToken.tokencode == flexer.getkeywordvalue("constraint"))
             {
                 //System.out.println(lcPPToken);
                 lcPPToken.tokencode = TBaseType.rw_constraint2;
             }
        }
    }

}

TSourceToken getprevtoken(TSourceToken ptoken){
   // ETokenType[] lcnonsolidtokenset = {ETokenType.ttwhitespace,ETokenType.ttreturn,ETokenType.ttsimplecomment,ETokenType.ttbracketedcomment} ;
    TSourceTokenList lcstlist = ptoken.container;
    if (TBaseType.assigned(lcstlist)){
        if ((ptoken.posinlist > 0)  && (lcstlist.size() > ptoken.posinlist-1))
        {
            if (!(
                    (lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttwhitespace)
                    ||(lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttreturn)
                    ||(lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttsimplecomment)
                    ||(lcstlist.get(ptoken.posinlist-1).tokentype ==  ETokenType.ttbracketedcomment)
                    )
                )
            {return lcstlist.get(ptoken.posinlist-1);}
            else{
            { return lcstlist.nextsolidtoken(ptoken.posinlist-1,-1,false);}
        }
        }
    }
    
    return null;
}

void doinformixtexttotokenlist(){
      TSourceToken lcprevtoken = null;
      int lcsteps = 0;
      TSourceToken asourcetoken,lctoken, lctoken2,lctoken3;
      int yychar;
      boolean iskeywordgo;

      asourcetoken = getanewsourcetoken();

      if ( asourcetoken == null ) return;

      yychar = asourcetoken.tokencode;


      boolean lcinopenrowset = false;
      int lcnested = 0;

      while (yychar > 0)
        {

          if ( asourcetoken.tokencode == TBaseType.rrw_openrowset )
            {
              // openrowset(....)
              lcinopenrowset = true;
              lcnested = 0;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttleftparenthesis)
            {
              if ( (lcsteps > 0) && TBaseType.assigned(lcprevtoken))
                {
                  if ( lcprevtoken.tokencode == TBaseType.rrw_primary )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 2;  //rw_primary2
                     //checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_foreign )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 4;  //rw_foreign2
                     //checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_unique )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 1;  //rw_unique2
                     // checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_distinct )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 6;  //rw_distinct2
                      //checkconstarinttoken(lcprevtoken);
                    }
                  lcprevtoken = null;
                  lcsteps = 0;
                }

              // openrowset(....)
              if ( lcinopenrowset )
                lcnested++;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttrightparenthesis)
            {
              // openrowset(....)
              if ( lcinopenrowset  )
                {
                  if ( (lcnested > 0) )
                    lcnested--;
                  if ( lcnested == 0 )
                    lcinopenrowset = false;
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttsemicolon )
            {
              if ( lcinopenrowset ){
                asourcetoken.tokentype = ETokenType.ttsemicolon2;
              }else{
                lctoken2 = asourcetoken.searchToken(TBaseType.rrw_begin,-1);
                  if (lctoken2 != null){
                     asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                     asourcetoken.tokentype = ETokenType.ttsemicolon3;
                  }else{
                      lctoken2 = asourcetoken.searchToken(TBaseType.rrw_try,-1);
                      if (lctoken2 == null){
                          lctoken2 = asourcetoken.searchToken(TBaseType.rrw_catch,-1);
                      }
                      if (lctoken2 != null){
                          lctoken3 = asourcetoken.searchToken(TBaseType.rrw_begin,-2);
                          if (lctoken3 != null){
                              asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                              asourcetoken.tokentype = ETokenType.ttsemicolon3;
                          }
                      }
                  }
              }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttperiod)
            {
              lctoken = getprevtoken(asourcetoken);
             //  System.out.println(lctoken);
              if ( TBaseType.assigned(lctoken) )
                { // go.fieldname, go is a table alias, not a go statement
                  if ( lctoken.tokencode == TBaseType.rrw_go )
                    {
                      lctoken.tokencode = TBaseType.ident;
                      lctoken.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_table ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_lock )
                  {
                      lctoken.tokencode = TBaseType.rw_locktable; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_to ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_connect )
                  {
                      lctoken.tokencode = TBaseType.rrw_informix_connect_to;//connect to statement
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_go )
            {
              iskeywordgo = true;
              lctoken = getprevtoken(asourcetoken);
              if ( TBaseType.assigned(lctoken) )
                { // go should not at same line as other sql statement
                  if ( lctoken.lineNo == asourcetoken.lineNo)
                    {
                      iskeywordgo = false;
                    }
                }

              if ( iskeywordgo )
                {
                  lcinopenrowset = false;
                  lcnested = 0;
                  lcprevtoken = asourcetoken;
                }
              else
                {
                  //  System.out.println(asourcetoken);
                  asourcetoken.tokencode = TBaseType.ident;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_primary )
            {
              // primary key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_foreign )
            {
              // foreign key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_unique )
            {
              // unique [clustered | nonclustered ] ( column list)
              lcsteps = 1;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.issolidtoken() )
            {
              if ( lcsteps > 0 )
                {
                  if (  !(TBaseType.mysametext("clustered",asourcetoken.astext)
                     || TBaseType.mysametext("nonclustered",asourcetoken.astext)) )
                    lcsteps--;
                }
            }

           //System.out.println(asourcetoken);
           sourcetokenlist.add(asourcetoken);


          //flexer.yylexwrap(asourcetoken);
          asourcetoken = getanewsourcetoken();
          if (asourcetoken != null){
              yychar = asourcetoken.tokencode;
          }else{
            yychar = 0;
          }

        }

  }

  void domssqlsqltexttotokenlist(){
      TSourceToken lcprevtoken = null;
      int lcsteps = 0;
      TSourceToken asourcetoken,lctoken, lctoken2,lctoken3;
      int yychar;
      boolean iskeywordgo;

      asourcetoken = getanewsourcetoken();
      
      if ( asourcetoken == null ) return;

      yychar = asourcetoken.tokencode;


      boolean lcinopenrowset = false;
      int lcnested = 0;

      while (yychar > 0)
        {

          if ( asourcetoken.tokencode == TBaseType.rrw_openrowset )
            {
              // openrowset(....)
              lcinopenrowset = true;
              lcnested = 0;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttleftparenthesis)
            {
              if ( (lcsteps > 0) && TBaseType.assigned(lcprevtoken))
                {
                  if ( lcprevtoken.tokencode == TBaseType.rrw_primary )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 2;  //rw_primary2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_foreign )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 4;  //rw_foreign2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_unique )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 1;  //rw_unique2
                      checkconstarinttoken(lcprevtoken);
                    }
                  lcprevtoken = null;
                  lcsteps = 0;
                }

              // openrowset(....)
              if ( lcinopenrowset )
                lcnested++;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttrightparenthesis)
            {
              // openrowset(....)
              if ( lcinopenrowset  )
                {
                  if ( (lcnested > 0) )
                    lcnested--;
                  if ( lcnested == 0 )
                    lcinopenrowset = false;
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttsemicolon )
            {
              if ( lcinopenrowset ){
                asourcetoken.tokentype = ETokenType.ttsemicolon2;
              }else{
                lctoken2 = asourcetoken.searchToken(TBaseType.rrw_begin,-1);
                  if (lctoken2 != null){
                     asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                     asourcetoken.tokentype = ETokenType.ttsemicolon3;
                  }else{
                      lctoken2 = asourcetoken.searchToken(TBaseType.rrw_try,-1);
                      if (lctoken2 == null){
                          lctoken2 = asourcetoken.searchToken(TBaseType.rrw_catch,-1);
                      }
                      if (lctoken2 != null){
                          lctoken3 = asourcetoken.searchToken(TBaseType.rrw_begin,-2);
                          if (lctoken3 != null){
                              asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                              asourcetoken.tokentype = ETokenType.ttsemicolon3;
                          }
                      }
                  }
              }

                lctoken = getprevtoken(asourcetoken);
                if ((lctoken != null)){
                    if (lctoken.tokentype == ETokenType.ttsemicolon){
                        // treat this semicolon as a whitespace
                        asourcetoken.tokencode = TBaseType.lexspace;
                    }
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttperiod)
            {
              lctoken = getprevtoken(asourcetoken);
             //  System.out.println(lctoken);
              if ( TBaseType.assigned(lctoken) )
                { // go.fieldname, go is a table alias, not a go statement
                  if ( lctoken.tokencode == TBaseType.rrw_go )
                    {
                      lctoken.tokencode = TBaseType.ident;
                      lctoken.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_table ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_lock )
                  {
                      lctoken.tokencode = TBaseType.rw_locktable; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_into ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_sqlserver_copy )
                  {
                     lctoken.tokencode = TBaseType.rrw_sqlserver_copyinto;
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_sqlserver_column ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_drop )
                  {
                      asourcetoken.tokencode = TBaseType.rrw_sqlserver_drop_column;
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_go )
            {
              iskeywordgo = true;
              lctoken = getprevtoken(asourcetoken);
              if ( TBaseType.assigned(lctoken) )
                { // go should not at same line as other sql statement
                  if (( lctoken.lineNo == asourcetoken.lineNo)&&(lctoken.tokencode != ';'))
                    {
                      iskeywordgo = false;
                    }
                }

              if ( iskeywordgo )
                {
                  lcinopenrowset = false;
                  lcnested = 0;
                  lcprevtoken = asourcetoken;
                }
              else
                {
                  //  System.out.println(asourcetoken);
                  asourcetoken.tokencode = TBaseType.ident;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_primary )
            {
              // primary key [clustered | nonclustered ] [hash] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_foreign )
            {
              // foreign key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_unique )
            {
              // unique [clustered | nonclustered ] [hash] ( column list)
              lcsteps = 1;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.issolidtoken() )
            {
              if ( lcsteps > 0 )
                {
                  if (  !(TBaseType.mysametext("clustered",asourcetoken.astext)
                     || TBaseType.mysametext("nonclustered",asourcetoken.astext)
                     || TBaseType.mysametext("hash",asourcetoken.astext)
                  ) )
                    lcsteps--;
                }
            }

           //System.out.println(asourcetoken); 
           sourcetokenlist.add(asourcetoken);


          //flexer.yylexwrap(asourcetoken);
          asourcetoken = getanewsourcetoken();
          if (asourcetoken != null){
              yychar = asourcetoken.tokencode;
          }else{
            yychar = 0;
          }

        }

  }

  void dosybasesqltexttotokenlist(){
      TSourceToken lcprevtoken = null;
      int lcsteps = 0;
      TSourceToken asourcetoken,lctoken, lctoken2,lctoken3;
      int yychar;
      boolean iskeywordgo;


      asourcetoken = getanewsourcetoken();

      if ( asourcetoken == null ) return;

      yychar = asourcetoken.tokencode;


      boolean lcinopenrowset = false;
      int lcnested = 0;

      while (yychar > 0)
        {

          if ( asourcetoken.tokencode == TBaseType.rrw_openrowset )
            {
              // openrowset(....)
              lcinopenrowset = true;
              lcnested = 0;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttleftparenthesis)
            {
              if ( (lcsteps > 0) && TBaseType.assigned(lcprevtoken))
                {
                  if ( lcprevtoken.tokencode == TBaseType.rrw_primary )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 2;  //rw_primary2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_foreign )
                    {
                     lcprevtoken.tokencode = TBaseType.rrw_select - 4;  //rw_foreign2
                     checkconstarinttoken(lcprevtoken);
                    }
                  else if ( lcprevtoken.tokencode == TBaseType.rrw_unique )
                    {
                      lcprevtoken.tokencode = TBaseType.rrw_select - 1;  //rw_unique2
                      checkconstarinttoken(lcprevtoken);
                    }
                  lcprevtoken = null;
                  lcsteps = 0;
                }

              // openrowset(....)
              if ( lcinopenrowset )
                lcnested++;
            }
          else if ( asourcetoken.tokentype == ETokenType.ttrightparenthesis)
            {
              // openrowset(....)
              if ( lcinopenrowset  )
                {
                  if ( (lcnested > 0) )
                    lcnested--;
                  if ( lcnested == 0 )
                    lcinopenrowset = false;
                }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttsemicolon )
            {
              if ( lcinopenrowset ){
                asourcetoken.tokentype = ETokenType.ttsemicolon2;
              }else{
                lctoken2 = asourcetoken.searchToken(TBaseType.rrw_begin,-1);
                  if (lctoken2 != null){
                     asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                     asourcetoken.tokentype = ETokenType.ttsemicolon3;
                  }else{
                      lctoken2 = asourcetoken.searchToken(TBaseType.rrw_try,-1);
                      if (lctoken2 == null){
                          lctoken2 = asourcetoken.searchToken(TBaseType.rrw_catch,-1);
                      }
                      if (lctoken2 != null){
                          lctoken3 = asourcetoken.searchToken(TBaseType.rrw_begin,-2);
                          if (lctoken3 != null){
                              asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
                              asourcetoken.tokentype = ETokenType.ttsemicolon3;
                          }
                      }
                  }
              }
            }
          else if ( asourcetoken.tokentype == ETokenType.ttperiod)
            {
              lctoken = getprevtoken(asourcetoken);
             //  System.out.println(lctoken);
              if ( TBaseType.assigned(lctoken) )
                { // go.fieldname, go is a table alias, not a go statement
                  if ( lctoken.tokencode == TBaseType.rrw_go )
                    {
                      lctoken.tokencode = TBaseType.ident;
                      lctoken.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_table ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_lock )
                  {
                      lctoken.tokencode = TBaseType.rw_locktable; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_sybase_isolation ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if (lctoken.tokencode == TBaseType.rrw_sybase_at )
                  {
                      lctoken.tokencode = TBaseType.rw_sybase_at1; //TBaseType.rw_locktable
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_update ){
              lctoken = getprevtoken(asourcetoken);
              if (TBaseType.assigned(lctoken))
              {
                  if ((lctoken.tokencode == TBaseType.rrw_not )
                          ||(lctoken.tokencode == TBaseType.rrw_and )
                          ||(lctoken.tokencode == TBaseType.rrw_or )
                          ||(lctoken.tokencode == TBaseType.rrw_if )
                      )
                  {
                      // this is update(column) in create trigger
                      asourcetoken.tokencode = TBaseType.rw_sybase_update1;
                  }
              }
          }
          else if ( asourcetoken.tokencode == TBaseType.rrw_go )
            {
              iskeywordgo = true;
              lctoken = getprevtoken(asourcetoken);
              if ( TBaseType.assigned(lctoken) )
                { // go should not at same line as other sql statement
                  if ( lctoken.lineNo == asourcetoken.lineNo)
                    {
                      iskeywordgo = false;
                    }
                }

              if ( iskeywordgo )
                {
                  lcinopenrowset = false;
                  lcnested = 0;
                  lcprevtoken = asourcetoken;
                }
              else
                {
                  //  System.out.println(asourcetoken);
                  asourcetoken.tokencode = TBaseType.ident;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
                }
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_primary )
            {
              // primary key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_foreign )
            {
              // foreign key [clustered | nonclustered ] ( column list)
              lcsteps = 2;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.tokencode == TBaseType.rrw_unique )
            {
              // unique [clustered | nonclustered ] ( column list)
              lcsteps = 1;
              lcprevtoken = asourcetoken;
            }
          else if ( asourcetoken.issolidtoken() )
            {
              if ( lcsteps > 0 )
                {
                  if (  !(TBaseType.mysametext("clustered",asourcetoken.astext)
                     || TBaseType.mysametext("nonclustered",asourcetoken.astext)) )
                    lcsteps--;
                }
            }

           //System.out.println(asourcetoken);
           sourcetokenlist.add(asourcetoken);


          //flexer.yylexwrap(asourcetoken);
          asourcetoken = getanewsourcetoken();
          if (asourcetoken != null){
              yychar = asourcetoken.tokencode;
          }else{
            yychar = 0;
          }

        }

  }

boolean IsValidPlaceForDivToSqlplusCmd(TSourceTokenList pstlist, int pPos){
   boolean ret = false;

    if ((pPos <= 0) || (pPos > pstlist.size() - 1 ) ) return ret;
    //tokentext directly before div must be ttreturn without space appending it
    TSourceToken lcst = pstlist.get(pPos - 1);
    if (lcst.tokentype != ETokenType.ttreturn) { return ret;}

    //I := Length(lcst.AsText);
    //if not (lcst.AsText[I] = ' ') then result := true;

    if (!(lcst.astext.charAt(lcst.astext.length()-1) == ' ')) {ret = true;}

    return ret;
}

// place holder function, always return false
boolean isvalidsqlpluscmdInPostgresql(String astr){
    return false;
}

//boolean isvalidsqlpluscmd(String astr){
//    boolean ret = false;
//    if (sqlpluskeywordList.size() == 0){
//    sqlpluskeywordList.put("ACC","1");
//        sqlpluskeywordList.put("ACCEPT","1");
//        sqlpluskeywordList.put("A","1");
//        sqlpluskeywordList.put("APPEND","1");
//        sqlpluskeywordList.put("ATTRIBUTE","1");
//        sqlpluskeywordList.put("BRE","1");
//        sqlpluskeywordList.put("BREAK","1");
//        sqlpluskeywordList.put("BTI","1");
//        sqlpluskeywordList.put("BTITLE","1");
//        sqlpluskeywordList.put("C","1");
//        sqlpluskeywordList.put("CHANGE","1");
//        sqlpluskeywordList.put("CL","1");
//        sqlpluskeywordList.put("CLEAR","1");
//        sqlpluskeywordList.put("COL","1");
//        sqlpluskeywordList.put("COLUMN","1");
//        sqlpluskeywordList.put("COMP","1");
//        sqlpluskeywordList.put("COMPUTE","1");
//        sqlpluskeywordList.put("CONN","1");
//        sqlpluskeywordList.put("CONNECT","1");
//        sqlpluskeywordList.put("COPY","1");
//        sqlpluskeywordList.put("DEF","1");
//        sqlpluskeywordList.put("DEFINE","1");
//        sqlpluskeywordList.put("DEL","1");
//        sqlpluskeywordList.put("DESC","1");
//        sqlpluskeywordList.put("DESCRIBE","1");
//        sqlpluskeywordList.put("DISC","1");
//        sqlpluskeywordList.put("DISCONNECT","1");
//        sqlpluskeywordList.put("ED","1");
//        sqlpluskeywordList.put("EDIT","1");
//        sqlpluskeywordList.put("EXEC","1");
//        sqlpluskeywordList.put("EXIT","1");
//        sqlpluskeywordList.put("QUIT","1");
//        sqlpluskeywordList.put("GET","1");
//        sqlpluskeywordList.put("HELP","1");
//        sqlpluskeywordList.put("HO","1");
//        sqlpluskeywordList.put("HOST","1");
//        sqlpluskeywordList.put("I","1");
//        sqlpluskeywordList.put("INPUT","1");
//        sqlpluskeywordList.put("L","1");
//        sqlpluskeywordList.put("LIST","1");
//        sqlpluskeywordList.put("PASSW","1");
//        sqlpluskeywordList.put("PASSWORD","1");
//        sqlpluskeywordList.put("PAU","1");
//        sqlpluskeywordList.put("PAUSE","1");
//        sqlpluskeywordList.put("PRI","1");
//        sqlpluskeywordList.put("PRINT","1");
//        sqlpluskeywordList.put("PROMPT","1");
//        sqlpluskeywordList.put("REM","1");
//        sqlpluskeywordList.put("REMARK","1");
//        sqlpluskeywordList.put("REPF","1");
//        sqlpluskeywordList.put("REPFOOTER","1");
//        sqlpluskeywordList.put("REPH","1");
//        sqlpluskeywordList.put("REPHEADER","1");
//        sqlpluskeywordList.put("R","1");
//        sqlpluskeywordList.put("RUN","1");
//        sqlpluskeywordList.put("SAV","1");
//        sqlpluskeywordList.put("SAVE","1");
//        sqlpluskeywordList.put("SET","1");
//        sqlpluskeywordList.put("SHO","1");
//        sqlpluskeywordList.put("SHOW","1");
//        sqlpluskeywordList.put("SPO","1");
//        sqlpluskeywordList.put("SPOOL","1");
//        sqlpluskeywordList.put("STA","1");
//        sqlpluskeywordList.put("START","1");
//        sqlpluskeywordList.put("STORE","1");
//        sqlpluskeywordList.put("TIMI","1");
//        sqlpluskeywordList.put("TIMING","1");
//        sqlpluskeywordList.put("TTI","1");
//        sqlpluskeywordList.put("TTITLE","1");
//        sqlpluskeywordList.put("UNDEF","1");
//        sqlpluskeywordList.put("UNDEFINE","1");
//        sqlpluskeywordList.put("VAR","1");
//        sqlpluskeywordList.put("VARIABLE","1");
//        sqlpluskeywordList.put("WHENEVER","1");
//        sqlpluskeywordList.put("@","1");
//        sqlpluskeywordList.put("@@","1");
//    }
//
//    try{
//     ret =   (Integer.parseInt( (String) sqlpluskeywordList.get(astr.toUpperCase()) ) ) == 1 ;
//    }catch(NumberFormatException e){
//        ret = false;
//    }
//
//    return ret;
//}

TSourceToken getprevsolidtoken(TSourceToken ptoken){
    TSourceToken ret = null;
    TSourceTokenList lctokenlist = ptoken.container;
    if (lctokenlist != null){
        if ((ptoken.posinlist > 0)  &&  (lctokenlist.size() > ptoken.posinlist-1))
        {
            if( !(
                    (lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttwhitespace)
                    ||(lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttreturn)
                ||(lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttsimplecomment)
                ||(lctokenlist.get(ptoken.posinlist-1).tokentype == ETokenType.ttbracketedcomment)
            ))
            { ret = lctokenlist.get(ptoken.posinlist-1);}
            else
            { ret = lctokenlist.nextsolidtoken(ptoken.posinlist-1,-1,false);}
        }
    }
    return ret;
}

    void doredshifttexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;


        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {


            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                    if (insqlpluscmd){
                        asourcetoken.insqlpluscmd = true;
                    }
                    break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                        isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                            lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.astext.equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                                // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                                && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.astext) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    if (asourcetoken.tokencode == TBaseType.rrw_redshift_rowtype) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }

                }
            }

            //flexer.yylexwrap(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if (asourcetoken != null){
                yychar = asourcetoken.tokencode;
            }else{
                yychar = 0;

                if (waitingreturnforfloatdiv)
                { // / at the end of line treat as sqlplus command
                    //isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                        lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }
            }

            if ((yychar == 0) && (prevst != null))
            {
            }

        }// while
    }

    void doansitexttotokenlist(){
        dodb2sqltexttotokenlist();
    }
    void dogaussdbtexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;

        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                    if (insqlpluscmd){
                        asourcetoken.insqlpluscmd = true;
                    }
                    break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                        isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                            lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.astext.equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                                && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.astext) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    // the inner keyword tokentext should be converted to TBaseType.ident when
                    // next solid tokentext is not join

                    if (prevst != null)
                    {
                        if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                        {
                            if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                            {prevst.tokencode = TBaseType.ident;}
                        }


                        if ((prevst.tokencode == TBaseType.rrw_not)
                                && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                        {
                            prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                        }

                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_inner)
                    {
                        prevst = asourcetoken;
                    }
                    else if (asourcetoken.tokencode == TBaseType.rrw_not)
                    {
                        prevst = asourcetoken;
                    }
                    else
                    { prevst = null;}

                    if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                            ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                    {
                        // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                        // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                        // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                        lcprevst = getprevsolidtoken(asourcetoken);
                        if (lcprevst != null)
                        {
                            if (lcprevst.tokencode == TBaseType.rrw_for)
                                lcprevst.tokencode = TBaseType.rw_for1;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                        //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                        TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                        if ( stKeep != null){
                            stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                        }
                    }

                    if ((asourcetoken.tokencode == TBaseType.rrw_postgresql_rowtype)||(asourcetoken.tokencode == TBaseType.rrw_postgresql_type)) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.JSON_EXIST) {
                        TSourceToken stPercent = asourcetoken.searchToken('=', -1);
                        if (stPercent != null) {    // = ?, ? after = should not be treated as json exist operator
                            asourcetoken.tokencode = TBaseType.ident;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_update) { // on conflict do update in insert statement
                        TSourceToken stDo = asourcetoken.searchToken(TBaseType.rrw_do, -1);
                        if (stDo != null) {
                            asourcetoken.tokencode = TBaseType.rrw_postgresql_do_update;
                        }
                    }


                }
            }

            //flexer.yylexwrap(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if (asourcetoken != null){
                yychar = asourcetoken.tokencode;
            }else{
                yychar = 0;

                if (waitingreturnforfloatdiv)
                { // / at the end of line treat as sqlplus command
                    //isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                        lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }

            }

            if ((yychar == 0) && (prevst != null))
            {
                if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
                { prevst.tokencode = TBaseType.ident;}
            }


        }


    }
    void dopostgresqltexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;

        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                    if (insqlpluscmd){
                        asourcetoken.insqlpluscmd = true;
                    }
                    break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                        isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                            lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.astext.equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                                // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                                && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.astext) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    // the inner keyword tokentext should be convert to TBaseType.ident when
                    // next solid tokentext is not join

                    if (prevst != null)
                    {
                        if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                        {
                            if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                            {prevst.tokencode = TBaseType.ident;}
                        }


                        if ((prevst.tokencode == TBaseType.rrw_not)
                                && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                        {
                            prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                        }

                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_inner)
                    {
                        prevst = asourcetoken;
                    }
                    else if (asourcetoken.tokencode == TBaseType.rrw_not)
                    {
                        prevst = asourcetoken;
                    }
                    else
                    { prevst = null;}

                    if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                            ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                    {
                        // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                        // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                        // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                        lcprevst = getprevsolidtoken(asourcetoken);
                        if (lcprevst != null)
                        {
                            if (lcprevst.tokencode == TBaseType.rrw_for)
                                lcprevst.tokencode = TBaseType.rw_for1;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                        //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                        TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                        if ( stKeep != null){
                            stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                        }
                    }

                    if ((asourcetoken.tokencode == TBaseType.rrw_postgresql_rowtype)||(asourcetoken.tokencode == TBaseType.rrw_postgresql_type)) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.JSON_EXIST) {
                        TSourceToken stPercent = asourcetoken.searchToken('=', -1);
                        if (stPercent != null) {    // = ?, ? after = should not be treated as json exist operator
                            asourcetoken.tokencode = TBaseType.ident;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_update) { // on conflict do update in insert statement
                        TSourceToken stDo = asourcetoken.searchToken(TBaseType.rrw_do, -1);
                        if (stDo != null) {
                            asourcetoken.tokencode = TBaseType.rrw_postgresql_do_update;
                        }
                    }


                }
            }

            //flexer.yylexwrap(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if (asourcetoken != null){
                yychar = asourcetoken.tokencode;
            }else{
                yychar = 0;

                if (waitingreturnforfloatdiv)
                { // / at the end of line treat as sqlplus command
                    //isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                        lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }

            }

            if ((yychar == 0) && (prevst != null))
            {
                if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
                { prevst.tokencode = TBaseType.ident;}
            }


        }


    }

void dosnowflakesqltexttotokenlist(){

    boolean insqlpluscmd = false;
    boolean isvalidplace = true;
    boolean waitingreturnforfloatdiv = false;
    boolean waitingreturnforsemicolon = false;
    boolean continuesqlplusatnewline = false;

    TSourceToken lct = null, prevst = null;

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
      {
        sourcetokenlist.add(asourcetoken);
        switch(yychar){
            case TBaseType.cmtdoublehyphen:
            case TBaseType.cmtslashstar:
            case TBaseType.lexspace:{
               if (insqlpluscmd){
                   asourcetoken.insqlpluscmd = true;
               }
               break;
            }
            case TBaseType.lexnewline:{
                if (insqlpluscmd){
                    insqlpluscmd = false;
                    isvalidplace = true;

                    if (continuesqlplusatnewline) {
                        insqlpluscmd = true;
                        isvalidplace = false;
                        asourcetoken.insqlpluscmd = true ;
                    }
                }

                if (waitingreturnforsemicolon){
                  isvalidplace = true;
                }
                if (waitingreturnforfloatdiv)
                {
                    isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                      lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }
                flexer.insqlpluscmd = insqlpluscmd;
                break;
            } //case newline
            default:{
                //solid tokentext
                continuesqlplusatnewline = false;
                waitingreturnforsemicolon = false;
                waitingreturnforfloatdiv = false;
                if (insqlpluscmd)
                {
                    asourcetoken.insqlpluscmd = true ;
                    if (asourcetoken.astext.equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                }
                else
                {
                    if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                    {waitingreturnforsemicolon = true;}
                    if ( (asourcetoken.tokentype == ETokenType.ttslash)
                      // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                       && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                    {
                        lct = asourcetoken;
                        waitingreturnforfloatdiv = true;
                    }
                    if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.astext) )
                    {
                        asourcetoken.tokencode = TBaseType.sqlpluscmd;
                        if (asourcetoken.tokentype != ETokenType.ttslash)
                        {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                        insqlpluscmd = true;
                        flexer.insqlpluscmd = insqlpluscmd;
                    }
                }
                isvalidplace = false;

                // the inner keyword tokentext should be convert to TBaseType.ident when
                // next solid tokentext is not join

                 if (prevst != null)
                 {
                    if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                    {
                        if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                        {prevst.tokencode = TBaseType.ident;}
                    }


                    if ((prevst.tokencode == TBaseType.rrw_not)
                    && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                    {
                          prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                    }

                 }

                if (asourcetoken.tokencode == TBaseType.rrw_inner)
                {
                    prevst = asourcetoken;
                }
                else if (asourcetoken.tokencode == TBaseType.rrw_not)
                {
                     prevst = asourcetoken;
                }
                else
                { prevst = null;}


            }
        }

         //flexer.yylexwrap(asourcetoken);
         asourcetoken = getanewsourcetoken();
         if (asourcetoken != null){
             yychar = asourcetoken.tokencode;
         }else{
           yychar = 0;

             if (waitingreturnforfloatdiv)
             { // / at the end of line treat as sqlplus command
                 //isvalidplace = true;
                 lct.tokencode = TBaseType.sqlpluscmd;
                 if (lct.tokentype != ETokenType.ttslash){
                   lct.tokentype = ETokenType.ttsqlpluscmd;
                 }
             }

         }

          if ((yychar == 0) && (prevst != null))
          {
              if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
              { prevst.tokencode = TBaseType.ident;}
          }


      } // while


}

    void doverticatexttotokenlist(){
        dopostgresqltexttotokenlist();
    }

    void docouchbasesqltexttotokenlist(){
        dopostgresqltexttotokenlist();
    }

    /**
     * Turn one token: `schema.table_name` into 3 tokens: `schema` . `table_name`
     * @param asourcetoken
     * @return
     */
    int splitQualifiedNameInBacktick(TSourceToken asourcetoken){
        int yychar = 0;

        List<String> elephantList = Arrays.asList(TBaseType.getTextWithoutQuoted(asourcetoken.toString()).split("\\."));
        int p = 0,offset=0;
        for(String s:elephantList){
            TSourceToken pst = new TSourceToken("`"+s+"`");
            pst.tokencode = asourcetoken.tokencode;
            pst.tokentype =  asourcetoken.tokentype;
            pst.tokenstatus = asourcetoken.tokenstatus;
            pst.lineNo = asourcetoken.lineNo;
            pst.columnNo = asourcetoken.columnNo+offset;
            if (p==0) offset++;// this count the first ` token
            offset = offset+s.length();
            pst.container = sourcetokenlist;
            if (p>0){// 第一个token使用被拆分前那个token的位置，从第二个开始的token，需要先把列表的位置指针加 1
                sourcetokenlist.curpos = sourcetokenlist.curpos+1;
            }
            pst.posinlist = sourcetokenlist.curpos;

            sourcetokenlist.add(pst);
            yychar = pst.tokencode;

            if (p != elephantList.size()-1){
                //`schema.table_name`, add period token in the middle of the backtick included identifier.
                TSourceToken periodst = new TSourceToken(".");
                periodst.tokencode = '.';
                periodst.tokentype =  ETokenType.ttperiod;
                periodst.tokenstatus = asourcetoken.tokenstatus;
                periodst.lineNo = asourcetoken.lineNo;
                periodst.columnNo = asourcetoken.columnNo + offset;
                offset++;
                periodst.container = sourcetokenlist;
                sourcetokenlist.curpos = sourcetokenlist.curpos+1;
                periodst.posinlist = sourcetokenlist.curpos;
                sourcetokenlist.add(periodst);
                yychar = periodst.tokencode;
            }

            p++;
            //System.out.println(s);
        }

        return yychar;

    }

    void dobigquerysqltexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;


        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            if (asourcetoken != null){
                sourcetokenlist.add(asourcetoken);
            }

            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            yychar = asourcetoken.tokencode;

            // `schema.table_name`
            if ((asourcetoken.tokencode == TBaseType.ident)
                    && (asourcetoken.toString().startsWith("`"))&& (asourcetoken.toString().endsWith("`"))
                    && (asourcetoken.toString().indexOf(".")>0)
            ){
                yychar = splitQualifiedNameInBacktick(asourcetoken);
                asourcetoken = null;
            }
        }
    }

    void dosoqlsqltexttotokenlist(){
        domssqlsqltexttotokenlist();
    }

void dogreenplumtexttotokenlist(){

        boolean insqlpluscmd = false;
        boolean isvalidplace = true;
        boolean waitingreturnforfloatdiv = false;
        boolean waitingreturnforsemicolon = false;
        boolean continuesqlplusatnewline = false;

        TSourceToken lct = null, prevst = null;

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
          {
            sourcetokenlist.add(asourcetoken);
            switch(yychar){
                case TBaseType.cmtdoublehyphen:
                case TBaseType.cmtslashstar:
                case TBaseType.lexspace:{
                   if (insqlpluscmd){
                       asourcetoken.insqlpluscmd = true;
                   }
                   break;
                }
                case TBaseType.lexnewline:{
                    if (insqlpluscmd){
                        insqlpluscmd = false;
                        isvalidplace = true;

                        if (continuesqlplusatnewline) {
                            insqlpluscmd = true;
                            isvalidplace = false;
                            asourcetoken.insqlpluscmd = true ;
                        }
                    }

                    if (waitingreturnforsemicolon){
                      isvalidplace = true;
                    }
                    if (waitingreturnforfloatdiv)
                    {
                        isvalidplace = true;
                        lct.tokencode = TBaseType.sqlpluscmd;
                        if (lct.tokentype != ETokenType.ttslash){
                          lct.tokentype = ETokenType.ttsqlpluscmd;
                        }
                    }
                    flexer.insqlpluscmd = insqlpluscmd;
                    break;
                } //case newline
                default:{
                    //solid tokentext
                    continuesqlplusatnewline = false;
                    waitingreturnforsemicolon = false;
                    waitingreturnforfloatdiv = false;
                    if (insqlpluscmd)
                    {
                        asourcetoken.insqlpluscmd = true ;
                        if (asourcetoken.astext.equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                    }
                    else
                    {
                        if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                        if ( (asourcetoken.tokentype == ETokenType.ttslash)
                          // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                           && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                        {
                            lct = asourcetoken;
                            waitingreturnforfloatdiv = true;
                        }
                        if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.astext) )
                        {
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }
                    }
                    isvalidplace = false;

                    // the inner keyword tokentext should be convert to TBaseType.ident when
                    // next solid tokentext is not join

                     if (prevst != null)
                     {
                        if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                        {
                            if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                            {prevst.tokencode = TBaseType.ident;}
                        }


                        if ((prevst.tokencode == TBaseType.rrw_not)
                        && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                        {
                              prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                        }

                     }

                    if (asourcetoken.tokencode == TBaseType.rrw_inner)
                    {
                        prevst = asourcetoken;
                    }
                    else if (asourcetoken.tokencode == TBaseType.rrw_not)
                    {
                         prevst = asourcetoken;
                    }
                    else
                    { prevst = null;}

                    if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                    ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                    {
                        // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                        // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                        // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                        lcprevst = getprevsolidtoken(asourcetoken);
                        if (lcprevst != null)
                        {
                            if (lcprevst.tokencode == TBaseType.rrw_for)
                              lcprevst.tokencode = TBaseType.rw_for1;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                        //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                        TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                        if ( stKeep != null){
                            stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                        }
                    }

                    if (asourcetoken.tokencode == TBaseType.rrw_greenplum_rowtype) {
                        TSourceToken stPercent = asourcetoken.searchToken('%', -1);
                        if (stPercent != null) {
                            stPercent.tokencode = TBaseType.rowtype_operator;
                        }
                    }


                }
            }

             //flexer.yylexwrap(asourcetoken);
             asourcetoken = getanewsourcetoken();
             if (asourcetoken != null){
                 yychar = asourcetoken.tokencode;
             }else{
               yychar = 0;

                 if (waitingreturnforfloatdiv)
                 { // / at the end of line treat as sqlplus command
                     //isvalidplace = true;
                     lct.tokencode = TBaseType.sqlpluscmd;
                     if (lct.tokentype != ETokenType.ttslash){
                       lct.tokentype = ETokenType.ttsqlpluscmd;
                     }
                 }

             }

              if ((yychar == 0) && (prevst != null))
              {
                  if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
                  { prevst.tokencode = TBaseType.ident;}
              }


          }


    }

void donetezzatexttotokenlist(){

    boolean insqlpluscmd = false;
    boolean isvalidplace = true;
    boolean waitingreturnforfloatdiv = false;
    boolean waitingreturnforsemicolon = false;
    boolean continuesqlplusatnewline = false;

    TSourceToken lct = null, prevst = null;

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
      {
        sourcetokenlist.add(asourcetoken);
        switch(yychar){
            case TBaseType.cmtdoublehyphen:
            case TBaseType.cmtslashstar:
            case TBaseType.lexspace:{
               if (insqlpluscmd){
                   asourcetoken.insqlpluscmd = true;
               }
               break;
            }
            case TBaseType.lexnewline:{
                if (insqlpluscmd){
                    insqlpluscmd = false;
                    isvalidplace = true;

                    if (continuesqlplusatnewline) {
                        insqlpluscmd = true;
                        isvalidplace = false;
                        asourcetoken.insqlpluscmd = true ;
                    }
                }

                if (waitingreturnforsemicolon){
                  isvalidplace = true;
                }
                if (waitingreturnforfloatdiv)
                {
                    isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                      lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }
                flexer.insqlpluscmd = insqlpluscmd;
                break;
            } //case newline
            default:{
                //solid tokentext
                continuesqlplusatnewline = false;
                waitingreturnforsemicolon = false;
                waitingreturnforfloatdiv = false;
                if (insqlpluscmd)
                {
                    asourcetoken.insqlpluscmd = true ;
                    if (asourcetoken.astext.equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                }
                else
                {
                    if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                    {waitingreturnforsemicolon = true;}
                    if ( (asourcetoken.tokentype == ETokenType.ttslash)
                      // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                       && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                    {
                        lct = asourcetoken;
                        waitingreturnforfloatdiv = true;
                    }
                    if ((isvalidplace) && isvalidsqlpluscmdInPostgresql(asourcetoken.astext) )
                    {
                        asourcetoken.tokencode = TBaseType.sqlpluscmd;
                        if (asourcetoken.tokentype != ETokenType.ttslash)
                        {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                        insqlpluscmd = true;
                        flexer.insqlpluscmd = insqlpluscmd;
                    }
                }
                isvalidplace = false;

                // the inner keyword tokentext should be convert to TBaseType.ident when
                // next solid tokentext is not join

                 if (prevst != null)
                 {
                    if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                    {
                        if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                        {prevst.tokencode = TBaseType.ident;}
                    }


                    if ((prevst.tokencode == TBaseType.rrw_not)
                    && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                    {
                          prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                    }

                 }

                if (asourcetoken.tokencode == TBaseType.rrw_inner)
                {
                    prevst = asourcetoken;
                }
                else if (asourcetoken.tokencode == TBaseType.rrw_not)
                {
                     prevst = asourcetoken;
                }
                else
                { prevst = null;}

                if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                {
                    // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                    // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                    // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        if (lcprevst.tokencode == TBaseType.rrw_for)
                          lcprevst.tokencode = TBaseType.rw_for1;
                    }
                }

                if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                    //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                    TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                    if ( stKeep != null){
                        stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                    }
                }


            }
        }

         //flexer.yylexwrap(asourcetoken);
         asourcetoken = getanewsourcetoken();
         if (asourcetoken != null){
             yychar = asourcetoken.tokencode;
         }else{
           yychar = 0;

             if (waitingreturnforfloatdiv)
             { // / at the end of line treat as sqlplus command
                 //isvalidplace = true;
                 lct.tokencode = TBaseType.sqlpluscmd;
                 if (lct.tokentype != ETokenType.ttslash){
                   lct.tokentype = ETokenType.ttsqlpluscmd;
                 }
             }

         }

          if ((yychar == 0) && (prevst != null))
          {
              if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
              { prevst.tokencode = TBaseType.ident;}
          }


      }
}

private boolean spaceAtTheEndOfReturnToken(String s){
    int pos = 0;
    if (s == null) return false;
    if (s.length() == 0) return false;

    return ((s.charAt(s.length() -1 ) == ' ')||(s.charAt(s.length() -1 ) == '\t'));
}

private int countLines(String s){
    int pos = 0, lf = 0, cr = 0;

    while (pos < s.length()) {
        if (s.charAt(pos) == '\r') {
            cr++;
            pos++;
            continue;
        }
        if (s.charAt(pos) == '\n') {
            lf++;
            pos++;
            continue;
        }

        if (s.charAt(pos) == ' ') {
            pos++;
            continue;
        }
        break;
    }

    if (lf >= cr) return lf;
    else return  cr;

 }

void dooraclesqltexttotokenlist(){

    boolean insqlpluscmd = false;
    boolean isvalidplace = true;
    boolean waitingreturnforfloatdiv = false;
    boolean waitingreturnforsemicolon = false;
    boolean continuesqlplusatnewline = false;

    ESqlPlusCmd currentCmdType = ESqlPlusCmd.spcUnknown;

    TSourceToken lct = null, prevst = null;

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
      {
        sourcetokenlist.add(asourcetoken);
        switch(yychar){
            case TBaseType.cmtdoublehyphen:
            case TBaseType.cmtslashstar:
            case TBaseType.lexspace:{
               if (insqlpluscmd){
                   asourcetoken.insqlpluscmd = true;
               }
               break;
            }
            case TBaseType.lexnewline:{
                if (insqlpluscmd){
                    insqlpluscmd = false;
                    isvalidplace = true;

                    if (continuesqlplusatnewline) {
                        insqlpluscmd = true;
                        isvalidplace = false;
                        asourcetoken.insqlpluscmd = true ;
                    }

                    if (!insqlpluscmd){
                        currentCmdType = ESqlPlusCmd.spcUnknown;
                    }
                }

                if (waitingreturnforsemicolon){
                  isvalidplace = true;
                }
                if (waitingreturnforfloatdiv)
                {
                    isvalidplace = true;
                    lct.tokencode = TBaseType.sqlpluscmd;
                    if (lct.tokentype != ETokenType.ttslash){
                      lct.tokentype = ETokenType.ttsqlpluscmd;
                    }
                }

                if (countLines(asourcetoken.toString()) > 1){
//                    SELECT *
//                            FROM foobar
//
//                    spool file

                    // there is a line after select, so spool is the right place to start a sqlplus command
                    isvalidplace = true;
                }

                flexer.insqlpluscmd = insqlpluscmd;
                break;
            } //case newline
            default:{
                //solid tokentext
                continuesqlplusatnewline = false;
                waitingreturnforsemicolon = false;
                waitingreturnforfloatdiv = false;
                if (insqlpluscmd)
                {
                    asourcetoken.insqlpluscmd = true ;
                    if (asourcetoken.astext.equalsIgnoreCase("-")) { continuesqlplusatnewline = true;}
                }
                else
                {
                    if (asourcetoken.tokentype == ETokenType.ttsemicolon)
                        {waitingreturnforsemicolon = true;}
                    if ( (asourcetoken.tokentype == ETokenType.ttslash)
                      // and (isvalidplace or sourcetokenlist.TokenBeforeCurToken(#10,false,false,false)) then
                       && (isvalidplace ||  (IsValidPlaceForDivToSqlplusCmd(sourcetokenlist,asourcetoken.posinlist)) ) )
                    {
                        lct = asourcetoken;
                        waitingreturnforfloatdiv = true;
                    }

                    currentCmdType =  TSqlplusCmdStatement.searchCmd(asourcetoken.astext,asourcetoken.nextToken());
                    // if (isvalidsqlpluscmd(asourcetoken.astext)){
                    if ( currentCmdType != ESqlPlusCmd.spcUnknown){
                        if (isvalidplace)
                        {

                            TSourceToken lnbreak = null;
                            boolean aRealSqlplusCmd = true;
                            if (sourcetokenlist.curpos >0){
                                lnbreak = sourcetokenlist.get(sourcetokenlist.curpos - 1);
                                aRealSqlplusCmd = !spaceAtTheEndOfReturnToken(lnbreak.toString());
                            }


//                            if ((countLines(lnbreak.toString()) > 1) && (!spaceAtTheEndOfReturnToken(lnbreak.toString()))) {
                            if (aRealSqlplusCmd) {
                                asourcetoken.prevTokenCode = asourcetoken.tokencode;
                                asourcetoken.tokencode = TBaseType.sqlpluscmd;
                                if (asourcetoken.tokentype != ETokenType.ttslash)
                                {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                                insqlpluscmd = true;
                                flexer.insqlpluscmd = insqlpluscmd;
                            }

                        }else if ((asourcetoken.tokencode == TBaseType.rrw_connect)&&(sourcetokenlist.returnbeforecurtoken(true))){
                            asourcetoken.tokencode = TBaseType.sqlpluscmd;
                            if (asourcetoken.tokentype != ETokenType.ttslash)
                            {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                            insqlpluscmd = true;
                            flexer.insqlpluscmd = insqlpluscmd;
                        }else if(sourcetokenlist.returnbeforecurtoken(true)){
                            TSourceToken lnbreak = sourcetokenlist.get(sourcetokenlist.curpos - 1);

                            if ((countLines(lnbreak.toString()) > 1) && (!spaceAtTheEndOfReturnToken(lnbreak.toString()))) {
                                asourcetoken.tokencode = TBaseType.sqlpluscmd;
                                if (asourcetoken.tokentype != ETokenType.ttslash)
                                {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                                insqlpluscmd = true;
                                flexer.insqlpluscmd = insqlpluscmd;
                            }
                        }
                    }

                }
                isvalidplace = false;

                // the inner keyword tokentext should be convert to TBaseType.ident when
                // next solid tokentext is not join

                 if (prevst != null)
                 {
                    if (prevst.tokencode == TBaseType.rrw_inner)//flexer.getkeywordvalue("INNER"))
                    {
                        if (asourcetoken.tokencode != flexer.getkeywordvalue("JOIN"))
                        {prevst.tokencode = TBaseType.ident;}
                    }else if ((prevst.tokencode == TBaseType.rrw_not)
                    && (asourcetoken.tokencode == flexer.getkeywordvalue("DEFERRABLE")))
                    {
                          prevst.tokencode = flexer.getkeywordvalue("NOT_DEFERRABLE");
                    }

                 }

                if (asourcetoken.tokencode == TBaseType.rrw_inner)
                {
                    prevst = asourcetoken;
                }
                else if (asourcetoken.tokencode == TBaseType.rrw_not)
                {
                     prevst = asourcetoken;
                }
                else
                { prevst = null;}

                if ((asourcetoken.tokencode == flexer.getkeywordvalue("DIRECT_LOAD"))
                ||  (asourcetoken.tokencode == flexer.getkeywordvalue("ALL")) )
                {
                    // RW_COMPRESS RW_FOR RW_ALL RW_OPERATIONS
                    // RW_COMPRESS RW_FOR RW_DIRECT_LOAD RW_OPERATIONS
                    // change rw_for to TBaseType.rw_for1, it conflicts with compress for update in create materialized view

                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        if (lcprevst.tokencode == TBaseType.rrw_for)
                          lcprevst.tokencode = TBaseType.rw_for1;
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_dense_rank){
                    //keep keyword can be column alias, make keep in keep_denserankclause as a different token code
                    TSourceToken stKeep  = asourcetoken.searchToken(TBaseType.rrw_keep,-2);
                    if ( stKeep != null){
                        stKeep.tokencode = TBaseType.rrw_keep_before_dense_rank;
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_full){
                    TSourceToken stMatch  = asourcetoken.searchToken(TBaseType.rrw_match,-1);
                    if (stMatch != null){   //match full
                        asourcetoken.tokencode = TBaseType.RW_FULL2;
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_join){
                    TSourceToken stFull  = asourcetoken.searchToken(TBaseType.rrw_full,-1);
                    if (stFull != null){ // full join
                        stFull.tokencode = TBaseType.RW_FULL2;
                    }else{
                        TSourceToken stNatural  = asourcetoken.searchToken(TBaseType.rrw_natural,-4);
                        if (stNatural != null){
                            stNatural.tokencode = TBaseType.RW_NATURAL2;
                        }
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_outer){
                    TSourceToken stFull  = asourcetoken.searchToken(TBaseType.rrw_full,-1);
                    if (stFull != null){ //full outer join
                        stFull.tokencode = TBaseType.RW_FULL2;
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_is){
                    TSourceToken stType  = asourcetoken.searchToken(TBaseType.rrw_type,-2);
                    if (stType != null){
                        stType.tokencode = TBaseType.rrw_type2;
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_as){
                    TSourceToken stType  = asourcetoken.searchToken(TBaseType.rrw_type,-2);
                    if (stType != null){
                        stType.tokencode = TBaseType.rrw_type2;
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_oid){
                    TSourceToken stType  = asourcetoken.searchToken(TBaseType.rrw_type,-2);
                    if (stType != null){
                        stType.tokencode = TBaseType.rrw_type2;
                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_type){
                    TSourceToken stPrev;
                    stPrev = asourcetoken.searchToken(TBaseType.rrw_drop,-1);
                    if (stPrev != null){
                        asourcetoken.tokencode = TBaseType.rrw_type2;
                    }
                    if (asourcetoken.tokencode == TBaseType.rrw_type){
                        stPrev  = asourcetoken.searchToken(TBaseType.rrw_of,-1);
                        if (stPrev != null){
                            asourcetoken.tokencode = TBaseType.rrw_type2;
                        }
                    }
                    if (asourcetoken.tokencode == TBaseType.rrw_type){
                        stPrev  = asourcetoken.searchToken(TBaseType.rrw_of,-1);
                        if (stPrev != null){
                            asourcetoken.tokencode = TBaseType.rrw_type2;
                        }
                    }
                    if (asourcetoken.tokencode == TBaseType.rrw_type){
                        stPrev  = asourcetoken.searchToken(TBaseType.rrw_create,-1);
                        if (stPrev != null){
                            asourcetoken.tokencode = TBaseType.rrw_type2;
                        }
                    }
                    if (asourcetoken.tokencode == TBaseType.rrw_type){
                        stPrev  = asourcetoken.searchToken(TBaseType.rrw_replace,-1);
                        if (stPrev != null){
                            asourcetoken.tokencode = TBaseType.rrw_type2;
                        }
                    }
                    if (asourcetoken.tokencode == TBaseType.rrw_type){
                        stPrev  = asourcetoken.searchToken('%',-1);
                        if (stPrev != null){
                            asourcetoken.tokencode = TBaseType.rrw_type2;
                        }
                    }
                }else  if ((asourcetoken.tokencode == TBaseType.rrw_by)||(asourcetoken.tokencode == TBaseType.rrw_to)){
                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        if ((lcprevst.tokencode == TBaseType.sqlpluscmd)&&(lcprevst.toString().equalsIgnoreCase("connect"))){
                            lcprevst.tokencode = TBaseType.rrw_connect;
                            lcprevst.tokentype = ETokenType.ttkeyword;
                            flexer.insqlpluscmd = false;

                            continuesqlplusatnewline = false;
                            waitingreturnforsemicolon = false;
                            waitingreturnforfloatdiv = false;
                            isvalidplace = false;
                            insqlpluscmd = false;
                        }

                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_with){ // rollback start in start with clause
                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        if ((lcprevst.tokencode == TBaseType.sqlpluscmd)&&(lcprevst.toString().equalsIgnoreCase("start"))){
                            lcprevst.tokencode = TBaseType.rrw_start;
                            lcprevst.tokentype = ETokenType.ttkeyword;
                            flexer.insqlpluscmd = false;

                            continuesqlplusatnewline = false;
                            waitingreturnforsemicolon = false;
                            waitingreturnforfloatdiv = false;
                            isvalidplace = false;
                            insqlpluscmd = false;
                        }

                    }
                }else  if (asourcetoken.tokencode == TBaseType.rrw_set){
                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        if (lcprevst.astext.equalsIgnoreCase("a")){
                            TSourceToken lcpp = getprevsolidtoken(lcprevst);
                            if (lcpp != null){
                                if ((lcpp.tokencode == TBaseType.rrw_not)||(lcpp.tokencode == TBaseType.rrw_is)){
                                    lcprevst.tokencode = TBaseType.rrw_oracle_a_in_aset;
                                    asourcetoken.tokencode = TBaseType.rrw_oracle_set_in_aset;
                                }
                            }
                        }
                    }

                }

            }
        }

         //flexer.yylexwrap(asourcetoken);
         asourcetoken = getanewsourcetoken();
         if (asourcetoken != null){
             yychar = asourcetoken.tokencode;
             if ((asourcetoken.tokencode == '.')&&(getprevsolidtoken(asourcetoken) != null)
                     &&((currentCmdType == ESqlPlusCmd.spcAppend)
                            ||(currentCmdType == ESqlPlusCmd.spcChange)||(currentCmdType == ESqlPlusCmd.spcInput)
                            ||(currentCmdType == ESqlPlusCmd.spcList)||(currentCmdType == ESqlPlusCmd.spcRun)
                        )){
//                 SELECT DISTINCT
//                 a.eca_usr_id,
//
//                         --        run_d,
//                         a.ent_rp_usr_id
//                 FROM t
                 // a.ent_rp_usr_id is not a real sqlplus command

                 TSourceToken lcprevst2 = getprevsolidtoken(asourcetoken);
                 lcprevst2.insqlpluscmd = false;
                 if (lcprevst2.prevTokenCode != 0){
                     lcprevst2.tokencode = lcprevst2.prevTokenCode;
                 }else{
                     lcprevst2.tokencode = TBaseType.ident;
                 }

                 flexer.insqlpluscmd = false;

                 continuesqlplusatnewline = false;
                 waitingreturnforsemicolon = false;
                 waitingreturnforfloatdiv = false;
                 isvalidplace = false;
                 insqlpluscmd = false;

                // System.out.println("Token before:"+lcprevst2.astext);
             }
         }else{
           yychar = 0;

             if (waitingreturnforfloatdiv)
             { // / at the end of line treat as sqlplus command
                 //isvalidplace = true;
                 lct.tokencode = TBaseType.sqlpluscmd;
                 if (lct.tokentype != ETokenType.ttslash){
                   lct.tokentype = ETokenType.ttsqlpluscmd;
                 }
             }

         }

          if ((yychar == 0) && (prevst != null))
          {
              if (prevst.tokencode == TBaseType.rrw_inner)// flexer.getkeywordvalue("RW_INNER"))
              { prevst.tokencode = TBaseType.ident;}
          }
      }
}

void dodb2sqltexttotokenlist(){
// treat echo, ! of db2 at the begin of each line as
// oracle sqlplus command

    boolean insqlpluscmd = false;
    boolean isvalidplace = true;
    int jdbc_escape_nest = 0;
//  isSolidTokenStart := false;

//  tempsourcetokenlist.clear;

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;


    while (yychar > 0)
    {

        switch (yychar){
          case TBaseType.lexnewline:
          {
              insqlpluscmd = false;
              isvalidplace = true;
              break;
          }
          case TBaseType.cmtdoublehyphen:
          {
              if (asourcetoken.astext.toLowerCase().indexOf("scriptoptions")>=0)
              {
                  asourcetoken.tokencode = TBaseType.scriptoptions;
                  asourcetoken.tokentype = ETokenType.ttidentifier;
              }

              if (insqlpluscmd)
              {
                  asourcetoken.insqlpluscmd = true;
              }
            break;
          }
          case TBaseType.rrw_jdbc_escape_fn: {
              jdbc_escape_nest++;
              asourcetoken.tokencode = TBaseType.lexspace;
            break;
          }
          case TBaseType.rrw_jdbc_escape_end: {
                jdbc_escape_nest--;
                asourcetoken.tokencode = TBaseType.lexspace;
                break;
          }
          default: //solid tokentext
          {
              if ((asourcetoken.tokencode == TBaseType.rrw_rr)
                    || (asourcetoken.tokencode ==  TBaseType.rrw_rs)
                    || (asourcetoken.tokencode ==  TBaseType.rrw_cs)
                    || (asourcetoken.tokencode ==  TBaseType.rrw_ur)
                  )
              {
                 // change with keyword in isolation clause to rrw_with_isolation
                  // so opt_restriction_clause in create view worked correctly
                  lcprevst = getprevsolidtoken(asourcetoken);
                  if (lcprevst != null)
                  {
                      if (lcprevst.tokencode == TBaseType.rrw_with)
                        lcprevst.tokencode = TBaseType.rrw_with_isolation;
                  }
              }

              if (insqlpluscmd)
              {
                  //asourcetoken.TokenCode := TBaseType.sqlpluscmd;
                  asourcetoken.insqlpluscmd = true;
              }
              else
              {
                  if (isvalidplace){
                   // System.out.println(asourcetoken.astext);
                  if (TBaseType.mycomparetext(asourcetoken.astext.toLowerCase(),"echo") == 0 )
                  {
                     // System.out.println("finded ");
                      asourcetoken.tokencode = TBaseType.sqlpluscmd;
                      if (asourcetoken.tokentype != ETokenType.ttslash)
                      {asourcetoken.tokentype = ETokenType.ttsqlpluscmd;}
                      asourcetoken.insqlpluscmd = true;
                      insqlpluscmd = true;
                  }
                }
              }
              isvalidplace = false;

              break;
          } //end of solid tokentext
        } //switch

//  TBaseType.lexnewline

        sourcetokenlist.add(asourcetoken);

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;

    } //while

}

    void doteradatatexttotokenlist(){

        TSourceToken asourcetoken,lcprevst;
        int yychar;

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;

            if ((asourcetoken.tokencode == TBaseType.rrw_casespecific)
                ||(asourcetoken.tokencode == TBaseType.rrw_teradata_cs))
            {
                    /* change not to not1 to make "not null column constriants" work correctly */
                //      |RW_NOT1 RW_CASESPECIFIC

                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == TBaseType.rrw_not)
                      lcprevst.tokencode = TBaseType.rw_not1;
                }
            } else if (asourcetoken.tokencode == ';'){
                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == ';')
                        asourcetoken.tokencode = TBaseType.sqlpluscmd;
                }
            } else if (asourcetoken.tokencode == TBaseType.variable){ // mantisbt/view.php?id=972
                if (asourcetoken.toString().toLowerCase().endsWith("begin")){ //MYPROC :BEGIN
                    asourcetoken.tokencode = TBaseType.rrw_begin;
                    asourcetoken.tokentype = ETokenType.ttkeyword;

                    lcprevst = getprevsolidtoken(asourcetoken);
                    if (lcprevst != null)
                    {
                        lcprevst.tokencode = TBaseType.mslabel;
                    }

                }
            }else if (asourcetoken.tokencode == TCustomLexer.UNICODE_ENCODE_ID){
                if (asourcetoken.toString().endsWith("008D")){ // REVERSE LINE FEED, https://codepoints.net/U+008D?lang=en
                    asourcetoken.tokencode = TBaseType.lexspace;
                }else{
                    asourcetoken.tokencode = TBaseType.ident;
                }
            }


            yychar = asourcetoken.tokencode;
        }

    }

void  docommonsqltexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}
    void  doodbcsqltexttotokenlist(){
        boolean insideODBC = false;
        TSourceToken odbcPrefix = null;
        domssqlsqltexttotokenlist();
        for (int i=0 ; i<sourcetokenlist.size();i++) {
            TSourceToken ast = sourcetokenlist.get(i);
            if ((ast.tokencode == '{')||(ast.tokencode == TBaseType.odbc_esc_prefix))
            {
                insideODBC = true;
                odbcPrefix = ast;
            }
            if ((ast.tokencode == '}')||(ast.tokencode == TBaseType.odbc_esc_terminator))
            {
                insideODBC = false;
                odbcPrefix = null;
            }
            if (((ast.tokencode == TBaseType.rrw_odbc_d)
                    ||(ast.tokencode == TBaseType.rrw_odbc_t)
                    ||(ast.tokencode == TBaseType.rrw_odbc_ts)
                    ||(ast.tokencode == TBaseType.rrw_odbc_fn)
                    ||(ast.tokencode == TBaseType.rrw_odbc_oj)
                )&&(!insideODBC)) {
                ast.tokencode = TBaseType.ident;
            }

            if ((ast.tokencode == TBaseType.rrw_call)&&(insideODBC)){
                odbcPrefix.setLinkToken(ast);
            }
        }
    }

void  dodaxsqltexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}


void  dohanasqltexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}

void  dohivetexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        if (asourcetoken != null){
            sourcetokenlist.add(asourcetoken);
        }
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        if (asourcetoken.tokencode == TBaseType.rrw_map){
            TSourceToken token = asourcetoken.searchToken(')',-1);
            if (token != null){
                asourcetoken.tokencode = TBaseType.ident;
            }
        }else if (asourcetoken.tokencode == '('){
//            TSourceToken token = asourcetoken.searchToken(TBaseType.ident,-1);
//            if (token != null){
//                token.tokencode = TBaseType.HIVE_FUNC_IDENT;
//            }
        }
        yychar = asourcetoken.tokencode;

        // `schema.table_name`
        if ((asourcetoken.tokencode == TBaseType.ident)
                && (asourcetoken.toString().startsWith("`"))&& (asourcetoken.toString().endsWith("`"))
                && (asourcetoken.toString().indexOf(".")>0)
        ){
            yychar = splitQualifiedNameInBacktick(asourcetoken);
            asourcetoken = null;
        }

    }

}

void  doimpalatexttotokenlist(){
    dohivetexttotokenlist();
}

void checkMySQLCommentToken(TSourceToken cmtToken){
//    if (cmtToken.tokencode == TBaseType.cmtdoublehyphen){
//        if ((cmtToken.toString().startsWith("--"))){
//            if (cmtToken.toString().length() > 2){
//                if (cmtToken.toString().charAt(2) != ' '){
//                   this.syntaxErrors.add(new TSyntaxError(cmtToken.toString(),
//                           cmtToken.lineNo,
//                           cmtToken.columnNo,
//                           "requires a space after the initial \"--\" ",EErrorType.sperror,10010
//                           ));
//                }
//            }
//            // there should be
//        }
//    }
}
    void dosparksqltexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size()-1).astext;
            }


            if (asourcetoken.tokencode == TBaseType.rrw_rollup)
            {
                //      with rollup

                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == TBaseType.rrw_with)
                        lcprevst.tokencode = TBaseType.with_rollup;
                }
            }

            yychar = asourcetoken.tokencode;
        }

    }

    void doathenatexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size()-1).astext;
            }


            yychar = asourcetoken.tokencode;
        }

    }

    void dodatabrickstexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size()-1).astext;
            }


            yychar = asourcetoken.tokencode;
        }

    }
    void doprestotexttotokenlist(){
        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;




        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size()-1).astext;
            }


            yychar = asourcetoken.tokencode;
        }

    }

void  domysqltexttotokenlist(){

        TSourceToken asourcetoken,lcprevst;
        int yychar;
        boolean startDelimiter = false;

        flexer.tmpDelimiter = "";

        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) return;
        yychar = asourcetoken.tokencode;
        checkMySQLCommentToken(asourcetoken);

        if ((asourcetoken.tokencode == TBaseType.rrw_mysql_delimiter)){
            startDelimiter = true;
        }

        while (yychar > 0)
        {
            sourcetokenlist.add(asourcetoken);
            asourcetoken = getanewsourcetoken();
            if ( asourcetoken == null ) break;
            checkMySQLCommentToken(asourcetoken);

            if ((asourcetoken.tokencode == TBaseType.lexnewline)&&(startDelimiter)){
                startDelimiter = false;
                flexer.tmpDelimiter  = sourcetokenlist.get(sourcetokenlist.size()-1).astext;
            }

            if ((asourcetoken.tokencode == TBaseType.rrw_mysql_delimiter)){
                startDelimiter = true;
            }

            if (asourcetoken.tokencode == TBaseType.rrw_rollup)
            {
                //      with rollup

                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode == TBaseType.rrw_with)
                      lcprevst.tokencode = TBaseType.with_rollup;
                }
            }

            if( (asourcetoken.tokencode == TBaseType.rrw_mysql_d)
                    ||(asourcetoken.tokencode == TBaseType.rrw_mysql_t)
                    ||(asourcetoken.tokencode == TBaseType.rrw_mysql_ts)
                    )
            {
                //      odbc date constant { d 'str' }
                lcprevst = getprevsolidtoken(asourcetoken);
                if (lcprevst != null)
                {
                    if (lcprevst.tokencode != '{')
                        asourcetoken.tokencode = TBaseType.ident;
                }
            }


            yychar = asourcetoken.tokencode;
        }

    }

void  doMdxtexttotokenlist(){

    TSourceToken asourcetoken,lcprevst;
    int yychar;

    asourcetoken = getanewsourcetoken();
    if ( asourcetoken == null ) return;
    yychar = asourcetoken.tokencode;

    while (yychar > 0)
    {
        sourcetokenlist.add(asourcetoken);
        asourcetoken = getanewsourcetoken();
        if ( asourcetoken == null ) break;
        yychar = asourcetoken.tokencode;
    }

}

void dosqltexttotokenlist(){
    switch(dbVendor){
        case dbvmssql:
        case dbvazuresql:
         {
            domssqlsqltexttotokenlist();
            break;
        }
        case dbvinformix:{
            doinformixtexttotokenlist();
            break;
        }
        case dbvsybase:{
            dosybasesqltexttotokenlist();
            break;
        }
        case dbvoracle:{
            dooraclesqltexttotokenlist();
            break;
        }
        case dbvdb2:{
            dodb2sqltexttotokenlist();
            break;
        }
        case dbvteradata:{
            doteradatatexttotokenlist();
            break;
        }
        case dbvpostgresql:{
            dopostgresqltexttotokenlist();
            break;
        }
        case dbvredshift:{
            doredshifttexttotokenlist();
            break;
        }
        case dbvgreenplum:{
            dogreenplumtexttotokenlist();
            break;
        }
        case dbvmdx: {
            doMdxtexttotokenlist();
            break;
        }
        case dbvnetezza:{
            donetezzatexttotokenlist();
            break;
        }
        case dbvhive:{
            dohivetexttotokenlist();
            break;
        }
        case dbvimpala:{
            doimpalatexttotokenlist();
            break;
        }
        case dbvmysql:{
            domysqltexttotokenlist();
            break;
        }
        case dbvhana:{
            dohanasqltexttotokenlist();
            break;
        }
        case dbvdax:{
            dodaxsqltexttotokenlist();
            break;
        }
        case dbvodbc:{
            doodbcsqltexttotokenlist();
            break;
        }
        case dbvvertica:{
            doverticatexttotokenlist();
            break;
        }
        case dbvopenedge:{
            domssqlsqltexttotokenlist();
            break;
        }
        case dbvcouchbase:{
            docouchbasesqltexttotokenlist();
            break;
        }
        case dbvsnowflake:{
            dosnowflakesqltexttotokenlist();
            break;
        }
        case dbvbigquery:{
            dobigquerysqltexttotokenlist();
            break;
        }
        case dbvsoql:{
            dosoqlsqltexttotokenlist();
            break;
        }
        case dbvsparksql:{
            dosparksqltexttotokenlist();
            break;
        }
        case dbvpresto:{
            doprestotexttotokenlist();
            break;
        }
        case dbvathena:{
            doathenatexttotokenlist();
            break;
        }
        case dbvdatabricks:{
            dodatabrickstexttotokenlist();
            break;
        }
        case dbvgaussdb:{
            dogaussdbtexttotokenlist();
            break;
        }
        case dbvansi:{
            doansitexttotokenlist();
            break;
        }
        default:{
            docommonsqltexttotokenlist();            
        }
    }

    doAfterTokenize();

    TBaseType.resetTokenChain(sourcetokenlist,0);

    processTokensInTokenTable(dbVendor);
    processTokensBeforeParse(dbVendor);

    closeFileStream();

    if (tokenListHandle != null){
        tokenListHandle.processTokenList(sourcetokenlist);
    }
}

void doAfterTokenize(){

    int leftParenCount = 0;
    int rightParenCount = 0;
    int leftIndex = 0;
    int rightIndex = sourcetokenlist.size() - 1;

    // Count opening parentheses at the beginning
    while (leftIndex < sourcetokenlist.size() && sourcetokenlist.get(leftIndex).tokencode == '(') {
        leftParenCount++;
        leftIndex++;
    }

    // Count closing parentheses at the end
    while (rightIndex >= 0 && sourcetokenlist.get(rightIndex).tokencode == ')') {
        rightParenCount++;
        rightIndex--;
    }

    // Set matching parentheses to be ignored
    int parensToIgnore = Math.min(leftParenCount, rightParenCount);
    // if there is a semicolon before the right parenthesis, set the semicolon to be ignored
    // mantisbt/view.php?id=3690

    if ((parensToIgnore > 0) && (sourcetokenlist.get(sourcetokenlist.size() - 1 - (parensToIgnore - 1) - 1).tokencode == ';')){
        // set to whitespace that this semicolon will be ignored during getting raw sql
        sourcetokenlist.get(sourcetokenlist.size() - 1 - (parensToIgnore - 1) - 1).tokentype = ETokenType.ttwhitespace;
        // set to ignore by yacc that this semicolon will be ignored during parsing
        sourcetokenlist.get(sourcetokenlist.size() - 1 - (parensToIgnore - 1) - 1).tokenstatus = ETokenStatus.tsignorebyyacc;
    }
//    for (int i = 0; i < parensToIgnore; i++) {
//        //sourcetokenlist.get(i).tokenstatus = ETokenStatus.tsignorebyyacc;
//        sourcetokenlist.get(i).tokencode = TBaseType.lexspace;
//
//       // sourcetokenlist.get(sourcetokenlist.size() - 1 - i).tokenstatus = ETokenStatus.tsignorebyyacc;
//        sourcetokenlist.get(sourcetokenlist.size() - 1 - i).tokencode = TBaseType.lexspace;
//    }

}



void processTokensBeforeParse(EDbVendor dbVendor){

        // 为确保性能，只在snowflake数据库中处理，因为目前只有snowflake数据库中有连续的分号有用户提出这个需求，其他数据库中暂时不处理
   if (dbVendor != EDbVendor.dbvsnowflake) return;

        // mantisbt/view.php?id=3579
        // if there are consecutive semicolon tokens, mark the second semi colon token as deleted token
    for(int i=0;i<sourcetokenlist.size();i++){
        TSourceToken st = sourcetokenlist.get(i);
        if (st.tokencode == ';'){
            TSourceToken nextToken = st.nextSolidToken();
            if (nextToken != null){
                if (nextToken.tokencode == ';'){
                    nextToken.tokenstatus = ETokenStatus.tsdeleted;
                }
            }
        }
    }

}

void processTokensInTokenTable(EDbVendor dbVendor){
    // 获得所有token后，根据需要对token code进行预处理， token table是 TBaseType.TOKEN_TABLE
     long[][] TOKEN_TABLE1 = flexer.TOKEN_TABLE;

    switch (dbVendor){
        case dbvbigquery:
        case dbvsnowflake:
            // case 1, DO 关键字如果没有发现对于的 FOR, WHILE 等关键字，把 DO 关键字的token code设置为 TBaseType.ident

            if (TOKEN_TABLE1[TBaseType.rrw_do][0] > 0){
                if ((TOKEN_TABLE1[TBaseType.rrw_while][0] == 0)&&(TOKEN_TABLE1[TBaseType.rrw_for][0] == 0)){
                    for(int i=0;i<sourcetokenlist.size();i++){
                        TSourceToken st = sourcetokenlist.get(i);
                        if (st.tokencode == TBaseType.rrw_do){
                            st.tokencode = TBaseType.ident;
                        }
                    }
                }
            }

            break;
    }

}

boolean isDollarFunctionDelimiter(int tokencode, EDbVendor dbVendor){
        return ((tokencode == TBaseType.rrw_postgresql_function_delimiter)&&(dbVendor == EDbVendor.dbvpostgresql))
                ||((tokencode == TBaseType.rrw_greenplum_function_delimiter)&&(dbVendor == EDbVendor.dbvgreenplum))
                ||((tokencode == TBaseType.rrw_redshift_function_delimiter)&&(dbVendor == EDbVendor.dbvredshift))
                ||((tokencode == TBaseType.rrw_snowflake_function_delimiter)&&(dbVendor == EDbVendor.dbvsnowflake));
}
void doongetrawsqlstatementevent(TCustomSqlStatement pcsqlstatement){
    pcsqlstatement.setGsqlparser(this);
    pcsqlstatement.parser = this.fparser;
    pcsqlstatement.plsqlparser = this.fplsqlparser;
    pcsqlstatement.setStartToken(pcsqlstatement.sourcetokenlist.get(0));
    pcsqlstatement.setEndToken(pcsqlstatement.sourcetokenlist.get(pcsqlstatement.sourcetokenlist.size()-1));
    sqlstatements.add(pcsqlstatement);

    if ((this.dbVendor == EDbVendor.dbvgaussdb) &&
            ((pcsqlstatement.sqlstatementtype == sstcreateprocedure)||(pcsqlstatement.sqlstatementtype == sstcreatefunction))){
        if (isOracleStyleRoutine){
            if (stFunction != null){
                stFunction.tokencode = TBaseType.GAUSSDB_FUNCTION_ORA;
            }else if (stProcedure != null){
                stProcedure.tokencode = TBaseType.GAUSSDB_PROCEDURE_ORA;
            }
        }
        stFunction = null;
        stProcedure = null;
        isOracleStyleRoutine = true; // 为下一次语句进行重置
    }

    // if stored procedure body is not written in sql or plsql, then, set the token in body to
    if (    ((this.dbVendor == EDbVendor.dbvpostgresql)||(this.dbVendor == EDbVendor.dbvgreenplum) ||(this.dbVendor == EDbVendor.dbvredshift) ||(this.dbVendor == EDbVendor.dbvsnowflake) )
             && (pcsqlstatement instanceof TRoutine)
       ){
         if (!((TRoutine)pcsqlstatement).isBodyInSQL()){
             TSourceToken st;
             boolean inBody = false;
             String routineBodyStr = "";
             for(int i=0;i<pcsqlstatement.sourcetokenlist.size();i++){
                 st = pcsqlstatement.sourcetokenlist.get(i);
                 if (isDollarFunctionDelimiter(st.tokencode,this.dbVendor)
//                         ((st.tokencode == TBaseType.rrw_postgresql_function_delimiter)&&(this.dbVendor == EDbVendor.dbvpostgresql))
//                        ||((st.tokencode == TBaseType.rrw_greenplum_function_delimiter)&&(this.dbVendor == EDbVendor.dbvgreenplum))
//                         ||((st.tokencode == TBaseType.rrw_redshift_function_delimiter)&&(this.dbVendor == EDbVendor.dbvredshift))
//                         ||((st.tokencode == TBaseType.rrw_snowflake_function_delimiter)&&(this.dbVendor == EDbVendor.dbvsnowflake))
                 ){
                     if (!inBody){
                         inBody = true;
                         routineBodyStr = st.toString();
                     }else{
                         inBody = false;
                         routineBodyStr += st.toString();
                         break;
                     }
                     continue;
                 }

                 if (inBody){
                     st.tokencode = TBaseType.sqlpluscmd;
                     routineBodyStr += st.toString();
                 }
             }

             ((TRoutine)pcsqlstatement).setRoutineBody(routineBodyStr);
         }
    }
}

    int doredshiftgetrawsqlstatements(){

        int waitingEnd = 0;
        boolean foundEnd = false, enterDeclare = false;



        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;


        if (isSinglePLBlock){
            gcurrentsqlstatement = new TCommonBlock(EDbVendor.dbvpostgresql);
        }

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (ast.tokencode == TBaseType.rrw_redshift_filter) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '(') {

                    }else{
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_redshift_array){
                TSourceToken st1 = ast.searchToken('<',1);
                if (st1 != null) { // array<varchar(20)>
                    ast.tokencode = TBaseType.rrw_redshift_array_type;
                }
            }
            else if (ast.tokencode == TBaseType.rrw_values) {
                TSourceToken stParen =  ast.searchToken('(',1);
                if (stParen != null){
                    TSourceToken stInsert  = ast.searchToken(TBaseType.rrw_insert,-ast.posinlist);
                    if (stInsert != null){
                        TSourceToken stSemiColon  = ast.searchToken(';',-ast.posinlist);
                        if ((stSemiColon != null)&&(stSemiColon.posinlist > stInsert.posinlist)){
//                            INSERT INTO test values (16,1), (8,2), (4,4), (2,0), (97, 16);
//                            VALUES (1);
                            // don't treat values(1) as insert values

                        }else{
                            TSourceToken stFrom  = ast.searchToken(TBaseType.rrw_from,-ast.posinlist);
                            if ((stFrom != null)&&(stFrom.posinlist > stInsert.posinlist)){
                                // don't treat values after from keyword as an insert values

                                // insert into inserttest values(10, 20, '40'), (-1, 2, DEFAULT),  ((select 2), (select i from (values(3) ) as foo (i)), 'values are fun!');

                            }else{
                                ast.tokencode = TBaseType.rrw_postgresql_insert_values;
                            }

                        }

                    }
                }
            }

            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd )
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        enterDeclare = false;
                        if (gcurrentsqlstatement.ispgplsql())
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);
                            foundEnd = false;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                            )
                            {
                                waitingEnd = 1;
                            }
                            else if (ast.tokencode == TBaseType.rrw_declare){
                                enterDeclare = true;
                            }
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsqlplus:{
                    if  (ast.insqlpluscmd)
                    {gcurrentsqlstatement.addtokentolist(ast);}
                    else
                    {
                        gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                    break;
                }//case stsqlplus

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if (ast.tokencode == TBaseType.rrw_redshift_function_delimiter){
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedurePgStartBody;

                        continue;
                    }

                    if (ast.tokencode == TBaseType.rrw_redshift_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());

                            }
                        }
                    }

                    if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (!enterDeclare))
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                    )
                    {
                        waitingEnd++;
                        enterDeclare = false;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_declare)
                    ){
                        enterDeclare = true;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0)
                                && (foundEnd)
                            //        && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())
                        ){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    if (( gst == EFindSqlStateType.ststoredprocedure ) && (ast.tokencode  == TBaseType.cmtdoublehyphen)){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    break;
                } //ststoredprocedure

                case ststoredprocedurePgStartBody:{
                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_redshift_function_delimiter){
                        gst = EFindSqlStateType.ststoredprocedurePgEndBody;
                        continue;
                    }

                    break;
                }

                case ststoredprocedurePgEndBody:{

                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    else if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_redshift_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                            }
                        }
                    }

                    break;
                }
            } //switch

        }//for


        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql)
                        || (gst == EFindSqlStateType.ststoredprocedure)
                        || (gst == EFindSqlStateType.ststoredprocedurePgEndBody)
                        ||(gst == EFindSqlStateType.sterror)||(isSinglePLBlock)
                ))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }

    int doverticagetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (ast.tokencode == TBaseType.rrw_date){
                TSourceToken st1 =  ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null){
                    if (st1.tokencode == '('){
                        ast.tokencode = TBaseType.rrw_vertica_date_function;
                    }
                }
            }else if ((ast.tokencode == TBaseType.rrw_vertica_greatest)||(ast.tokencode == TBaseType.rrw_vertica_least)){
                TSourceToken st1 =  ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null){
                    if (st1.tokencode != '('){
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }

            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd )
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a token text to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        if (gcurrentsqlstatement.isverticaplsql())
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);
                            foundEnd = true;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                                    )
                            { waitingEnd = 1;}
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo),"Error when tokenlize"
                                , EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsqlplus:{
                    if  (ast.insqlpluscmd)
                    {gcurrentsqlstatement.addtokentolist(ast);}
                    else
                    {
                        gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                    break;
                }//case stsqlplus

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                            )
                    {
                        waitingEnd++;
                        foundEnd = false;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (foundEnd) && (gcurrentsqlstatement.VerticaStatementCanBeSeparatedByBeginEndPair())){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    break;
                } //ststoredprocedure
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                        (gst == EFindSqlStateType.sterror)))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }

    boolean checkTokenPairWithEnd(int tokencode){
        return    ((tokencode == TBaseType.rrw_if)||(tokencode == TBaseType.rrw_case)
                ||(tokencode == TBaseType.rrw_loop)||(tokencode == TBaseType.rrw_repeat)
                ||(tokencode == TBaseType.rrw_while)||(tokencode == TBaseType.rrw_for)
                ||(tokencode == TBaseType.rrw_case)
        );
    }

    int dobigquerygetrawsqlstatements() {
        int errorcount = 0;
        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        int i,c, beginNested = 0,waitingEnd=0;
        TSourceToken ast = null, lcprevsolidtoken = null;
        boolean waitingDelimiter = false;

        int waitingEnds[] = new int[stored_procedure_nested_level];
        stored_procedure_type sptype[] = new stored_procedure_type[stored_procedure_nested_level];
        stored_procedure_status procedure_status[] = new stored_procedure_status[stored_procedure_nested_level];
        boolean endBySlashOnly = true;
        int nestedProcedures = 0, nestedParenthesis = 0;


        //reset delimiter
        userDelimiterStr = defaultDelimiterStr;

        for (i=0;i < sourcetokenlist.size();i++)
        {
            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (ast.tokencode == TBaseType.rrw_bigquery_struct){
                TSourceToken st1 =  ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null){
                    if (st1.tokencode == '('){
                        ast.tokencode = TBaseType.rrw_bigquery_struct_constructor;
                    }
                }
            } else if ((ast.tokencode == TBaseType.sconst)
            //        ||(ast.tokencode == '+')||(ast.tokencode == '-')
            ){
            if (TTypeName.searchTypeByName(lcprevsolidtoken.toString()) != null){
                // date '2021-2-1', turn date to TBaseType.rrw_bigquery_datatype_used_to_cast
                if (lcprevsolidtoken.tokencode != TBaseType.rrw_interval){
                        lcprevsolidtoken.tokencode = TBaseType.rrw_bigquery_datatype_used_to_cast;
                    }
                }
//            else if (ast.tokencode == TBaseType.rrw_timestamp){
//                TSourceToken st1 =  ast.nextSolidToken();
//                if (st1 != null){
//                    if (st1.tokencode == TBaseType.sconst){
//                        ast.tokencode = TBaseType.rrw_bigquery_timestamp_before_const;
//                    }
//                }
            }else if((ast.tokencode == TBaseType.rrw_time)||(ast.tokencode == TBaseType.rrw_date)) {

                TSourceToken st1 = ast.nextSolidToken();
                if (st1 != null) {
                    if (st1.tokencode == TBaseType.sconst) {
                        // ast.tokencode = TBaseType.rrw_bigquery_time_before_const;
                    } else if (st1.tokencode == '(') {
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_from){
                TSourceToken st1 =  ast.prevSolidToken();
                if (st1 != null){ // select i.from as `from` from t, treats from in i.from as identifier
                    if (st1.tokencode == '.'){
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }

            switch(gst){
                case sterror:
                {
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                }
                case stnormal :
                {
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (TBaseType.assigned(gcurrentsqlstatement))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        continue;
                    }


                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (TBaseType.assigned(gcurrentsqlstatement))
                    {
                        if (gcurrentsqlstatement.isBigQueryplsql())
                        {
                            nestedProcedures = 0;
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);

                            switch (gcurrentsqlstatement.sqlstatementtype){
                                case sstplsql_createprocedure:
                                case sstcreateprocedure:
                                    sptype[nestedProcedures] = stored_procedure_type.procedure;
                                    break;
                                case sstplsql_createfunction:
                                    sptype[nestedProcedures] = stored_procedure_type.function;
                                    break;
                                case sstplsql_createpackage:
                                    sptype[nestedProcedures] = stored_procedure_type.package_spec;
                                    if (ast.searchToken(TBaseType.rrw_body,5) != null){
                                        sptype[nestedProcedures] = stored_procedure_type.package_body;
                                    }
                                    break;
                                case sst_plsql_block:
                                    sptype[nestedProcedures] = stored_procedure_type.block_with_declare;
                                    if (ast.tokencode == TBaseType.rrw_begin) {
                                        sptype[nestedProcedures] = stored_procedure_type.block_with_begin;
                                    }
                                    break;
                                case sstplsql_createtrigger:
                                    sptype[nestedProcedures] = stored_procedure_type.create_trigger;
                                    break;
                                case sstoraclecreatelibrary:
                                    sptype[nestedProcedures] = stored_procedure_type.create_library;
                                    break;
                                case sstplsql_createtype_placeholder:
                                    gst = EFindSqlStateType.stsql;
                                    break;
                                default:
                                    sptype[nestedProcedures] = stored_procedure_type.others;
                                    break;
                            }

                            if (sptype[0] == stored_procedure_type.block_with_declare){
                                // sd
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.is_as;
                            }else if (sptype[0] == stored_procedure_type.block_with_begin){
                                // sb
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.body;
                            }else if (sptype[0] == stored_procedure_type.procedure){
                                // ss
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.start;
                            }else if (sptype[0] == stored_procedure_type.function){
                                // ss
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.start;
                            }else if (sptype[0] == stored_procedure_type.package_spec){
                                // ss
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.start;
                            }else if (sptype[0] == stored_procedure_type.package_body){
                                // ss
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.start;
                            }else if (sptype[0] == stored_procedure_type.create_trigger){
                                // ss
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.start;
                                //procedure_status[0] = stored_procedure_status.body;
                            }else if (sptype[0] == stored_procedure_type.create_library){
                                // ss
                                endBySlashOnly = false;
                                procedure_status[0] = stored_procedure_status.bodyend;
                            }else {
                                // so
                                endBySlashOnly = true;
                                procedure_status[0] = stored_procedure_status.bodyend;
                            }
                            //foundEnd = false;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    //||(ast.tokencode == TBaseType.rrw_procedure)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                            )
                            {
                                //waitingEnd = 1;
                                waitingEnds[nestedProcedures] = 1;
                            }

                        }else if ((gcurrentsqlstatement.sqlstatementtype == sst_ifstmt)
                            ||(gcurrentsqlstatement.sqlstatementtype == sst_loopstmt)
                            ||(gcurrentsqlstatement.sqlstatementtype == sstRepeat)
                            ||(gcurrentsqlstatement.sqlstatementtype == sstWhilestmt)
                            ||(gcurrentsqlstatement.sqlstatementtype == sstForStmt)
                            ||(gcurrentsqlstatement.sqlstatementtype == sst_plsql_block)
                                ||(gcurrentsqlstatement.sqlstatementtype == sst_casestmt)

                        ){
                            gst = EFindSqlStateType.stBigQueryIf;
                            waitingEnd = 1;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                    }


                    if (!TBaseType.assigned(gcurrentsqlstatement) ) //error tokentext found
                    {

//                        errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                        errorcount = 1;
                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }
                    break;
                }
                case stBigQueryIf:{
                    gcurrentsqlstatement.addtokentolist(ast);

                    if  (checkTokenPairWithEnd(ast.tokencode))
                    { // if... end if
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (ast.tokencode == TBaseType.rrw_end)
                    { // if ... end if
                        TSourceToken next = ast.nextSolidToken();
                        if (next != null){
                            if (checkTokenPairWithEnd(next.tokencode)){ // if ... end if;
                                waitingEnd--;
                            }else if (next.tokencode == ';'){ // begin ... end ;
                                waitingEnd--;
                            }

                        }
                    }
                    else if   (
                            (ast.tokencode == ';')&&(waitingEnd == 0)
                    ){
                        gst = EFindSqlStateType.stnormal;

                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    break;
                }
                case stsqlplus:{
                    if (ast.tokencode  == TBaseType.lexnewline){
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }else {
                        {gcurrentsqlstatement.addtokentolist(ast);}
                    }

                    break;
                }//case source command or \. command
                case stsql :
                {
                    if ((ast.tokentype == ETokenType.ttsemicolon)&&(gcurrentsqlstatement.sqlstatementtype != ESqlStatementType.sstmysqldelimiter))
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    if (ast.toString().equalsIgnoreCase(userDelimiterStr))
                    {
                        gst = EFindSqlStateType.stnormal;
                        ast.tokencode = ';';// treat it as semicolon
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);

                    if ((ast.tokencode  == TBaseType.lexnewline)
                            &&(gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmysqldelimiter)){
                        gst = EFindSqlStateType.stnormal;
                        userDelimiterStr = "";
                        for(int k=0;k<gcurrentsqlstatement.sourcetokenlist.size();k++){
                            TSourceToken st = gcurrentsqlstatement.sourcetokenlist.get(k);
                            if ((st.tokencode == TBaseType.rrw_mysql_delimiter)
                                    ||(st.tokencode == TBaseType.lexnewline)
                                    ||(st.tokencode == TBaseType.lexspace))
                            {
                                continue;
                            }

                            userDelimiterStr += st.toString();
                            // System.out.println("userDelimiterStr:"+userDelimiterStr);
                            // System.out.print(gcurrentsqlstatement.sourcetokenlist.get(k).toString());
                        }
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        continue;
                    }

                    break;
                }
                case ststoredprocedure :
                {
                    if (procedure_status[nestedProcedures] != stored_procedure_status.bodyend){
                        gcurrentsqlstatement.addtokentolist(ast);
                    }

                    switch (procedure_status[nestedProcedures]){
                        case start:
                            if ((ast.tokencode == TBaseType.rrw_as)||(ast.tokencode == TBaseType.rrw_is)){
                                // s1
                                if (sptype[nestedProcedures] != stored_procedure_type.create_trigger){
                                    if ((sptype[0] == stored_procedure_type.package_spec) && (nestedProcedures > 0)){
                                        //when it's a package specification, only top level accept as/is
                                    }else{
                                        procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                                        if (ast.searchToken("language",1) != null){
                                            // if as language is used in create function, then switch state to stored_procedure_status.body directly.
//                                        CREATE OR REPLACE FUNCTION THING.addressparse(p_addressline1 VARCHAR2) RETURN VARCHAR2 AUTHID DEFINER
//                                        as Language JAVA NAME 'AddressParser.parse(java.lang.String) return java.lang.String';
//                                        /
                                            if (nestedProcedures == 0){
                                                //  procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                                gst = EFindSqlStateType.stsql;
                                            }else{
                                                procedure_status[nestedProcedures] = stored_procedure_status.body;
                                                nestedProcedures--;
                                                //if (nestedProcedures > 0){ nestedProcedures--;}
                                            }

                                        }
                                    }
                                }
                            }else if(ast.tokencode == TBaseType.rrw_begin){
                                // s4
                                if (sptype[nestedProcedures] == stored_procedure_type.create_trigger) waitingEnds[nestedProcedures]++;

                                if (nestedProcedures > 0) {nestedProcedures--;}
                                procedure_status[nestedProcedures] = stored_procedure_status.body;
                                waitingEnds[nestedProcedures] = 1;
                            }else if(ast.tokencode == TBaseType.rrw_end){
                                //s10
                                if((nestedProcedures > 0)&&(waitingEnds[nestedProcedures - 1] == 1)
                                        &&((sptype[nestedProcedures - 1] == stored_procedure_type.package_body)
                                        ||(sptype[nestedProcedures - 1] == stored_procedure_type.package_spec))){
                                    nestedProcedures--;
                                    procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                }
                            }else if((ast.tokencode == TBaseType.rrw_procedure)||(ast.tokencode == TBaseType.rrw_function)){
                                //s3
                                if((nestedProcedures > 0)&&(waitingEnds[nestedProcedures] == 0)
                                        &&(procedure_status[nestedProcedures - 1] == stored_procedure_status.is_as)){
                                    nestedProcedures--;
                                    nestedProcedures++;
                                    waitingEnds[nestedProcedures] = 0;
                                    procedure_status[nestedProcedures] = stored_procedure_status.start;
                                }
                            }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger) && (ast.tokencode == TBaseType.rrw_declare) )
                            {
                                procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                            }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger)&&(ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) )
                            {
                                // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                                ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);

                                //make / a sqlplus cmd
                                gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger) )
                            {
                                if (ast.tokencode == TBaseType.rrw_trigger){
                                    TSourceToken compoundSt =  ast.searchToken(TBaseType.rrw_oracle_compound, -1);
                                    if (compoundSt != null){
                                        //it's trigger with compound trigger block
                                        procedure_status[nestedProcedures] = stored_procedure_status.body;
                                        waitingEnds[nestedProcedures]++;
                                    }
                                }
                            }else if ((sptype[nestedProcedures] == stored_procedure_type.function)&&(ast.tokencode == TBaseType.rrw_teradata_using) )
                            {
                                if ((ast.searchToken("aggregate",-1) != null)||(ast.searchToken("pipelined",-1) != null)){
                                    if (nestedProcedures == 0){
                                        //  procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                        gst = EFindSqlStateType.stsql;
                                    }else{
                                        procedure_status[nestedProcedures] = stored_procedure_status.body;
                                        nestedProcedures--;
                                    }
                                }

                            }else{
                                //other tokens, do nothing
                            }
                            break;
                        case is_as:
                            if((ast.tokencode == TBaseType.rrw_procedure)||(ast.tokencode == TBaseType.rrw_function)){
                                // s2
                                nestedProcedures++;
                                waitingEnds[nestedProcedures] = 0;
                                procedure_status[nestedProcedures] = stored_procedure_status.start;

                                if (nestedProcedures > stored_procedure_nested_level - 1){
                                    gst = EFindSqlStateType.sterror;
                                    nestedProcedures--;
                                }

                            }else if(ast.tokencode == TBaseType.rrw_begin){
                                // s5
                                if ((nestedProcedures == 0)&&
                                        ((sptype[nestedProcedures]==stored_procedure_type.package_body)
                                                ||(sptype[nestedProcedures]==stored_procedure_type.package_spec))){
                                    //top level package or package body's BEGIN keyword already count,
                                    // so don't increase waitingEnds[nestedProcedures] here

                                }else {
                                    waitingEnds[nestedProcedures]++;
                                }
                                procedure_status[nestedProcedures] = stored_procedure_status.body;
                            }else if(ast.tokencode == TBaseType.rrw_end){
                                // s6
                                if ((nestedProcedures == 0)&&(waitingEnds[nestedProcedures] == 1)&&
                                        ((sptype[nestedProcedures]==stored_procedure_type.package_body)||(sptype[nestedProcedures]==stored_procedure_type.package_spec))){
                                    procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                    waitingEnds[nestedProcedures]--;
                                }else{
                                    waitingEnds[nestedProcedures]--;
                                }
                            }else if(ast.tokencode == TBaseType.rrw_case)
                            {
                                if (ast.searchToken(';',1) == null){
                                    //this is not case before ;
                                    waitingEnds[nestedProcedures]++;
                                }
                            }
                            else {
                                //other tokens, do nothing
                            }
                            break;
                        case body:
                            if ((ast.tokencode == TBaseType.rrw_begin))
                            {
                                waitingEnds[nestedProcedures]++;
                            } else if (ast.tokencode == TBaseType.rrw_if)
                            {

                                if (ast.searchToken(';',2) == null){
                                    //this is not if before ;

                                    // 2015-02-27, change 1 to 2 make it able to detect label name after case
                                    // like this: END CASE l1;
                                    waitingEnds[nestedProcedures]++;
                                }
                            } else if(ast.tokencode == TBaseType.rrw_case)
                            {
//                            if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                //this is not case after END
//                             waitingEnds[nestedProcedures]++;
//                            }
                                if (ast.searchToken(';',2) == null){
                                    //this is not case before ;
                                    if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                                        waitingEnds[nestedProcedures]++;
                                    }
                                }
                            } else if((ast.tokencode == TBaseType.rrw_loop)
                                        ||(ast.tokencode == TBaseType.rrw_while)||(ast.tokencode == TBaseType.rrw_repeat)
                                    ||(ast.tokencode == TBaseType.rrw_for)
                            )
                            {
                                if (!((ast.searchToken(TBaseType.rrw_end,-1) != null)
                                        &&(ast.searchToken(';',2) != null))){
                                    // exclude loop like this:
                                    // end loop [labelname];
                                    waitingEnds[nestedProcedures]++;
                                }

                            }else if (ast.tokencode == TBaseType.rrw_end){
                                //foundEnd = true;
                                waitingEnds[nestedProcedures]--;
                                //if (waitingEnd < 0) { waitingEnd = 0;}
                                if (waitingEnds[nestedProcedures] == 0){
                                    if (nestedProcedures == 0){
                                        // s7
                                        procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                    }else {
                                        // s71
                                        nestedProcedures--;
                                        procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                                    }
                                }
                            }else  if ((waitingEnds[nestedProcedures] == 0)&&(ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                            {
                                //sql ref: c:\prg\gsqlparser\Test\TestCases\oracle\createtrigger.sql, line 53
                                ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);

                                //make / a sqlplus cmd
                                gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            }
                            break;
                        case bodyend:
                            if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                            {
                                // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                                ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);

                                //make / a sqlplus cmd
                                gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.tokencode == ';') )
                            {    // single dot at a seperate line
                              //  ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);

                            } else  if ((ast.searchToken(TBaseType.rrw_package,1) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_procedure,1) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_function,1) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_create,1) != null)&&(ast.searchToken(TBaseType.rrw_package,4) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_create,1) != null)&&(ast.searchToken(TBaseType.rrw_library,4) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_alter,1) != null)&&(ast.searchToken(TBaseType.rrw_trigger,2) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_select,1) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_commit,1) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            } else  if ((ast.searchToken(TBaseType.rrw_grant,1) != null)&&
                                    (ast.searchToken(TBaseType.rrw_execute,2) != null)&&(!endBySlashOnly))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stnormal;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                            }else {
                                gcurrentsqlstatement.addtokentolist(ast);
                            }
                            break;
                        case end:
                            break;
                        default:
                            break;
                    }


                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else if (ast.tokentype == ETokenType.ttslash){
                            ast.tokencode = '/';
                        }
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    final int wrapped_keyword_max_pos = 20;
                    if ((ast.tokencode == TBaseType.rrw_wrapped)&&(ast.posinlist - gcurrentsqlstatement.sourcetokenlist.get(0).posinlist  < wrapped_keyword_max_pos)){
                        if (gcurrentsqlstatement instanceof TCommonStoredProcedureSqlStatement){
                            ((TCommonStoredProcedureSqlStatement)gcurrentsqlstatement).setWrapped(true);
                        }

                        if (gcurrentsqlstatement instanceof TPlsqlCreatePackage){
                            if (ast.prevSolidToken() != null){
                                ((TPlsqlCreatePackage) gcurrentsqlstatement).setPackageName(fparser.getNf().createObjectNameWithPart(ast.prevSolidToken()));
                            }
                        }
                    }

                    break;
                } //ststoredprocedure

            } //case
        } //for


        //last statement
        if (TBaseType.assigned(gcurrentsqlstatement) &&
                ( (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure)
                    || (gst == EFindSqlStateType.stBigQueryIf)|| (gst == EFindSqlStateType.sterror)
                )
        )
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return errorcount;
    }


    int docouchbasegetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

//            if (ast.tokencode == ':'){
//                TSourceToken st1 = ast.searchToken(TBaseType.ident,-1);
//                if (st1 != null){
//                    st1.tokencode = TBaseType.IDENT_BEFORE_COLON;
//                }
//            }

            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd )
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a token text to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {

                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsqlplus:{
                    if  (ast.insqlpluscmd)
                    {gcurrentsqlstatement.addtokentolist(ast);}
                    else
                    {
                        gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                    break;
                }//case stsqlplus

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                            )
                    {
                        waitingEnd++;
                        foundEnd = false;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (foundEnd) && (gcurrentsqlstatement.VerticaStatementCanBeSeparatedByBeginEndPair())){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    break;
                } //ststoredprocedure
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                        (gst == EFindSqlStateType.sterror)))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }
    int doathenagetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;


            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }


                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        if (gcurrentsqlstatement.isathenaplsql())
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);
                            foundEnd = false;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                            )
                            { waitingEnd = 1;}
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                    )
                    {
                        waitingEnd++;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0)
                                && (foundEnd)
                            //        && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())
                        ){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    break;
                } //ststoredprocedure
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                        (gst == EFindSqlStateType.sterror)))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }
    int dodatabricksgetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false;
        EDataType tmpDatatype = null;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (ast.tokencode == TBaseType.rrw_values) {
                TSourceToken stParen =  ast.searchToken('(',1);
                if (stParen != null){
                    TSourceToken stInsert  = ast.searchToken(TBaseType.rrw_insert,-ast.posinlist,';',true);
                    if (stInsert != null){
                        TSourceToken stSemiColon  = ast.searchToken(';',-ast.posinlist);
                        if ((stSemiColon != null)&&(stSemiColon.posinlist > stInsert.posinlist)){
    //                            INSERT INTO test values (16,1), (8,2), (4,4), (2,0), (97, 16);
    //                            VALUES (1);
                            // don't treat values(1) as insert values

                        }else{
                            TSourceToken stFrom  = ast.searchToken(TBaseType.rrw_from,-ast.posinlist,';',true);
                            if (stFrom != null){
                                // don't treat values after from keyword as an insert values

                                // insert into inserttest values(10, 20, '40'), (-1, 2, DEFAULT),  ((select 2), (select i from (values(3) ) as foo (i)), 'values are fun!');

                                // let check the INSERT keyword is close to VALUES than FROM keyword, if yes, treat it as insert values

                                if (stInsert.posinlist > stFrom.posinlist){
                                    // https://www.sqlparser.com/bugs/mantisbt/view.php?id=3354
                                    ast.tokencode = TBaseType.rrw_databricks_values_insert;
                                }

                            }else{
                                ast.tokencode = TBaseType.rrw_databricks_values_insert;
                            }
                        }
                    }
                }
           }else if ((ast.tokencode == TBaseType.sconst)||(ast.tokencode == '+')||(ast.tokencode == '-')){
                if (TTypeName.searchTypeByName(lcprevsolidtoken.toString()) != null){
                    // date '2021-2-1', turn date to TBaseType.rrw_databricks_datatype_used_to_cast
                    if (lcprevsolidtoken.tokencode != TBaseType.rrw_interval){
                        lcprevsolidtoken.tokencode = TBaseType.rrw_databricks_datatype_used_to_cast;
                    }
                }
            }


            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }


                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        if (gcurrentsqlstatement.isdatabricksplsql())
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);
                            foundEnd = false;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                            )
                            { waitingEnd = 1;}
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                    )
                    {
                        waitingEnd++;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0)
                                && (foundEnd)
                            //        && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())
                        ){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    break;
                } //ststoredprocedure
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                        (gst == EFindSqlStateType.sterror)))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }
    int doprestogetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;


            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }


                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        if (gcurrentsqlstatement.isprestoplsql())
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);
                            foundEnd = false;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                            )
                            { waitingEnd = 1;}
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                    )
                    {
                        waitingEnd++;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0)
                                && (foundEnd)
                            //        && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())
                        ){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    break;
                } //ststoredprocedure
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                        (gst == EFindSqlStateType.sterror)))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }

    // isSinglePLBlock = true, 表示 netezza 中的 declear..begen/end blcok 重新在 postgresql 中解析，只需要看做一个整体即可，不需要再逐个语句进行区分。
    // TCreateProcedureStmt.java, private void doParseFunctionBody(EDbVendor dbVendor,String inputSQL,boolean isSQLBlock){

    private boolean isSinglePLBlock = false;

    public void setSinglePLBlock(boolean singlePLBlock) {
        isSinglePLBlock = singlePLBlock;
    }

    // In gaussDB, used to find pkg name during getRawsqlstatement()
    // mode = 1, pkg spec, mode = 2, pkg body, mode = 3, pkg name after end and before ;
    String findPkgName(TSourceToken st, int mode){

        String pkgName="";

        boolean inpkgname = false;
        if (mode == 3) inpkgname = true;
        int MAX_TRY_TOKENS = 20;
        int tryTokens = 0;
        while (st != null){

            if (st.isnonsolidtoken()) {
                st = st.getNextTokenInChain();
                continue;
            }
            tryTokens++;

            if ((mode == 1 ) && (st.tokencode == TBaseType.rrw_package)){
                inpkgname = true;
                st = st.getNextTokenInChain();
                continue;
            }else if ((mode == 2 ) && (st.tokencode == TBaseType.rrw_body)){
                inpkgname = true;
                st = st.getNextTokenInChain();
                continue;
            }

            if(((mode == 1)||(mode == 2))
                && ((st.tokencode == TBaseType.rrw_is)||(st.tokencode == TBaseType.rrw_as))
                ){
                break;
            }else if ((mode == 3) && (st.tokencode == ';')){
                break;
            }

            if (!inpkgname) {
                st = st.getNextTokenInChain();
                continue;
            }
            pkgName = pkgName+st.toString();
            st = st.getNextTokenInChain();

            if (tryTokens > MAX_TRY_TOKENS){
                TBaseType.log(String.format("Package name is not found in create package"),TLog.ERROR,st);
                break;
            }
        }

        return pkgName;
    }

    boolean isOracleStyleRoutine = true;
    TSourceToken stProcedure = null;
    TSourceToken stFunction = null;

    int doansigetrawsqlstatements(){
        return dodb2getrawsqlstatements();
    }

    int dogaussdbgetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false, enterDeclare = false;
        isOracleStyleRoutine = true;
        stProcedure = null;
        stFunction = null;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

//        if (isSinglePLBlock){
//            gcurrentsqlstatement = new TCommonBlock(EDbVendor.dbvgaussdb);
//        }

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;
            if (isSinglePLBlock){
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }

            if (ast.tokencode == TBaseType.JSON_EXIST){
                TSourceToken stConstant  = ast.searchToken(TBaseType.sconst,1);
                if (stConstant == null){
                    ast.tokencode = TBaseType.ident;
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_POSITION) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '(') {
                        ast.tokencode = TBaseType.rrw_postgresql_POSITION_FUNCTION;
                    }
                }
            }else  if (ast.tokencode == TBaseType.rrw_postgresql_ordinality) {
                TSourceToken lcprevst = getprevsolidtoken(ast);

                if (lcprevst != null) {
                    if (lcprevst.tokencode == TBaseType.rrw_with) {
                        TSourceToken lcnextst = ast.nextSolidToken();
                        if ((lcnextst != null)&&(lcnextst.tokencode == TBaseType.rrw_as)){
                            //  with ordinality as (select 1 as x) select * from ordinality;
                            // don't change with to rrw_postgresql_with_lookahead
                        }else{
                            lcprevst.tokencode = TBaseType.rrw_postgresql_with_lookahead;
                        }

                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_filter) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '(') {

                    }else{
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_jsonb) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '?') {
                        st1.tokencode = TBaseType.OP_JSONB_QUESTION;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.GAUSSDB_TO_NUMBER) {
                TSourceToken st1 = ast.searchToken('(',1);
                if (st1 == null) {
                    ast.tokencode = TBaseType.ident;
                }
            }
            else if (ast.tokencode == '?') {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == TBaseType.sconst) {
                        ast.tokencode = TBaseType.OP_JSONB_QUESTION;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_values) {
                TSourceToken stParen =  ast.searchToken('(',1);
                if (stParen != null){
                    TSourceToken stInsert  = ast.searchToken(TBaseType.rrw_insert,-ast.posinlist);
                    if (stInsert != null){
                        TSourceToken stSemiColon  = ast.searchToken(';',-ast.posinlist);
                        if ((stSemiColon != null)&&(stSemiColon.posinlist > stInsert.posinlist)){
//                            INSERT INTO test values (16,1), (8,2), (4,4), (2,0), (97, 16);
//                            VALUES (1);
                            // don't treat values(1) as insert values

                        }else{
                            TSourceToken stFrom  = ast.searchToken(TBaseType.rrw_from,-ast.posinlist);
                            if ((stFrom != null)&&(stFrom.posinlist > stInsert.posinlist)){
                                // 不但要发现 from keywords, 而且该keyword 不能出现在 insert 前
                                // 避免这个文件中的 insert values 部分内容被视为 subquery. "C:\prg\gsp_sqlfiles\TestCases\private\java\gaussdb\insert_in_procedure.sql"

                                // don't treat values after from keyword as an insert values

                                // insert into inserttest values(10, 20, '40'), (-1, 2, DEFAULT),  ((select 2), (select i from (values(3) ) as foo (i)), 'values are fun!');

                            }else{
                                ast.tokencode = TBaseType.rrw_postgresql_insert_values;
                            }

                        }

                    }
                }
            }

            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd )
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        enterDeclare = false;
                        if (gcurrentsqlstatement.isGaussDBStoredProcedure())
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstplsql_createpackage){
                                gst = EFindSqlStateType.stGaussDBPkgSpec;
                                TPlsqlCreatePackage pkgspec = (TPlsqlCreatePackage) gcurrentsqlstatement;
                                pkgspec.setPackageNameStr(findPkgName(ast.getNextTokenInChain(),1));
                                //System.out.println("pkg spec name: "+ pkgspec.getPackageNameStr());
                            }else if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstoraclecreatepackagebody){
                                gst = EFindSqlStateType.stGaussDBPkgBody;
                                TPlsqlCreatePackage pkgspec = (TPlsqlCreatePackage) gcurrentsqlstatement;
                                pkgspec.setPackageNameStr(findPkgName(ast.getNextTokenInChain(),2));
                                //System.out.println("pkg body name: "+ pkgspec.getPackageNameStr());
                            }else{
                                gst = EFindSqlStateType.ststoredprocedure;

                                foundEnd = false;
                                if   ((ast.tokencode == TBaseType.rrw_begin)
                                        ||(ast.tokencode == TBaseType.rrw_package)
                                        || (ast.searchToken(TBaseType.rrw_package,4) != null)
                                )
                                {
                                    waitingEnd = 1;
                                }
                                else if (ast.tokencode == TBaseType.rrw_declare){
                                    enterDeclare = true;
                                }

                                if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreateprocedure)
                                {
                                    // create procedure, create or replace procedure
                                    stProcedure = ast.searchToken(TBaseType.rrw_procedure,3);
                                }
                                else if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatefunction){
                                    // create function, create or replace function
                                    stFunction = ast.searchToken(TBaseType.rrw_function,3);
                                }
                            }
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsqlplus:{
                    if  (ast.insqlpluscmd)
                    {gcurrentsqlstatement.addtokentolist(ast);}
                    else
                    {
                        gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                    break;
                }//case stsqlplus

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken()) //most probably is / cmd
                    {
                        // gaussdb 中的普通sql（非存储过程）中如果出现 / ,并且被识别为 sqlplus, 应该是误判，例如下面的sql，需要把 / 复原为普通的字符 /
                        //                        select CASE WHEN 1 IS NULL Then -1
                        //                        Else
                        //                        date_part('day' ,NVL(sysdate, TO_DATE('4712-12-31','YYYY-MM-DD')) - NVL(sysdate, TO_DATE('1900-01-01' ,'YYYY-MM-DD')))
                        //                                /
                        //                                NVL(NVL(null,1), -1) END AS varCompressionRate
                        TSourceToken st = ast.nextSolidToken();
                        if (st.tokentype == ETokenType.ttslash){
                            st.tokencode = '/';
                        }else { // 非 / 的 sqlplus 命令仍然作为sqlplus处理，但 gaussdb 中是否存在类似 oracle的sqlplus，需要进一步验证
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if (ast.tokencode == TBaseType.rrw_postgresql_function_delimiter){
                        gcurrentsqlstatement.addtokentolist(ast);
                        isOracleStyleRoutine = false;
                        gst = EFindSqlStateType.ststoredprocedurePgStartBody;
                        continue;
                    }

                    if (ast.tokencode == TBaseType.rrw_postgresql_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                                isOracleStyleRoutine = false;
                            }
                        }
                    }

                    if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (!enterDeclare))
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;

                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                    )
                    {
                        waitingEnd++;
                        enterDeclare = false;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_declare)
                            ||(ast.tokencode == TBaseType.rrw_as)||(ast.tokencode == TBaseType.rrw_is)
                    ){
                        TSourceToken next = ast.nextSolidToken();
                        if ((next != null) && ((next.tokencode == TBaseType.sconst)||(next.tokencode == TBaseType.GAUSSDB_NULL))){
                                // CREATE FUNCTION func_add_sql(integer, integer) RETURNS integer
                                //    AS 'select $1 + $2;'
                                //    LANGUAGE SQL
                                //    IMMUTABLE
                                //    RETURNS NULL ON NULL INPUT;
                        }else{
                            //
//                            create or replace function func_001(a in out pkg_type.table_of_index_int, b in out pkg_type.table_of_index_var) --#add in & inout #defult value
//                            return pkg_type.table_of_index_int
//                            as
//                            table_of_index_int_val pkg_type.table_of_index_int;
//                            table_of_index_var_val pkg_type.table_of_index_var;
//                            begin
//                            for i in 1..2 loop
                            enterDeclare = true;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                    ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;

                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;

                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (!enterDeclare)
                                && (foundEnd)
                            //        && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())
                        ){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    if (( gst == EFindSqlStateType.ststoredprocedure ) && (ast.tokencode  == TBaseType.cmtdoublehyphen)){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    break;
                } //ststoredprocedure

                case ststoredprocedurePgStartBody:{
                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_postgresql_function_delimiter){
                        gst = EFindSqlStateType.ststoredprocedurePgEndBody;
                        continue;
                    }

                    break;
                }

                case ststoredprocedurePgEndBody:{

                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    else if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_postgresql_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                            }
                        }
                    }

                    break;
                }

                case stGaussDBPkgSpec:{
                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_end){
                        TPlsqlCreatePackage plsqlCreatePackage = (TPlsqlCreatePackage)gcurrentsqlstatement;

                        String pkgName = findPkgName(ast.getNextTokenInChain(),3);
                        if ((pkgName != null)&&( pkgName.equalsIgnoreCase(plsqlCreatePackage.getPackageNameStr()))) {
                            gst = EFindSqlStateType.stGaussDBPkgSpecEnd;
                        }
                    }
                    break;
                }
                case stGaussDBPkgSpecEnd:{
                    gcurrentsqlstatement.addtokentolist(ast);
                    if (ast.tokencode == ';'){
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    break;
                }

                case stGaussDBPkgBody:{
                    gcurrentsqlstatement.addtokentolist(ast);
                    if (ast.tokencode == TBaseType.rrw_end){
                        TPlsqlCreatePackage plsqlCreatePackage = (TPlsqlCreatePackage)gcurrentsqlstatement;

                        String pkgName = findPkgName(ast.getNextTokenInChain(),3);
                        if ((pkgName != null)&&( pkgName.equalsIgnoreCase(plsqlCreatePackage.getPackageNameStr()))) {
                            gst = EFindSqlStateType.stGaussDBPkgBodyEnd;
                        }
                    }
                    break;
                }
                case stGaussDBPkgBodyEnd:{
                    gcurrentsqlstatement.addtokentolist(ast);
                    if (ast.tokencode == ';'){
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    break;
                }

            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql)
                        || (gst == EFindSqlStateType.ststoredprocedure)
                        || (gst == EFindSqlStateType.ststoredprocedurePgEndBody)
                        ||(gst == EFindSqlStateType.sterror)||(isSinglePLBlock)
                ))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }

    int dopostgresqlgetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false, enterDeclare = false;



        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        if (isSinglePLBlock){
            gcurrentsqlstatement = new TCommonBlock(EDbVendor.dbvpostgresql);
        }

        for (int i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;
            if (isSinglePLBlock){
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }

            if (ast.tokencode == TBaseType.JSON_EXIST){
                TSourceToken stConstant  = ast.searchToken(TBaseType.sconst,1);
                if (stConstant == null){
                    ast.tokencode = TBaseType.ident;
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_POSITION) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '(') {
                        ast.tokencode = TBaseType.rrw_postgresql_POSITION_FUNCTION;
                    }
                }
            }else  if (ast.tokencode == TBaseType.rrw_postgresql_ordinality) {
                TSourceToken lcprevst = getprevsolidtoken(ast);

                if (lcprevst != null) {
                    if (lcprevst.tokencode == TBaseType.rrw_with) {
                        TSourceToken lcnextst = ast.nextSolidToken();
                        if ((lcnextst != null)&&(lcnextst.tokencode == TBaseType.rrw_as)){
                            //  with ordinality as (select 1 as x) select * from ordinality;
                            // don't change with to rrw_postgresql_with_lookahead
                        }else{
                            lcprevst.tokencode = TBaseType.rrw_postgresql_with_lookahead;
                        }

                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_filter) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '(') {

                    }else{
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_postgresql_jsonb) {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == '?') {
                        st1.tokencode = TBaseType.OP_JSONB_QUESTION;
                    }
                }
            }
            else if (ast.tokencode == '?') {
                TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null) {
                    if (st1.tokencode == TBaseType.sconst) {
                        ast.tokencode = TBaseType.OP_JSONB_QUESTION;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_values) {
                TSourceToken stParen =  ast.searchToken('(',1);
                if (stParen != null){
                    TSourceToken stInsert  = ast.searchToken(TBaseType.rrw_insert,-ast.posinlist);
                    if (stInsert != null){
                        TSourceToken stSemiColon  = ast.searchToken(';',-ast.posinlist);
                        if ((stSemiColon != null)&&(stSemiColon.posinlist > stInsert.posinlist)){
//                            INSERT INTO test values (16,1), (8,2), (4,4), (2,0), (97, 16);
//                            VALUES (1);
                            // don't treat values(1) as insert values

                        }else{
                            TSourceToken stFrom  = ast.searchToken(TBaseType.rrw_from,-ast.posinlist);
                            if ((stFrom != null)&&(stFrom.posinlist > stInsert.posinlist)){
                                // don't treat values after from keyword as an insert values

                                // insert into inserttest values(10, 20, '40'), (-1, 2, DEFAULT),  ((select 2), (select i from (values(3) ) as foo (i)), 'values are fun!');

                            }else{
                                ast.tokencode = TBaseType.rrw_postgresql_insert_values;
                            }

                        }

                    }
                }
            }

            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (gcurrentsqlstatement != null)
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                        {
                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                        }

                        continue;
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd )
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        enterDeclare = false;
                        if (gcurrentsqlstatement.ispgplsql())
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            gcurrentsqlstatement.addtokentolist(ast);
                            foundEnd = false;
                            if   ((ast.tokencode == TBaseType.rrw_begin)
                                    ||(ast.tokencode == TBaseType.rrw_package)
                                    || (ast.searchToken(TBaseType.rrw_package,4) != null)
                                    )
                            {
                                waitingEnd = 1;
                            }
                            else if (ast.tokencode == TBaseType.rrw_declare){
                                enterDeclare = true;
                            }
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsqlplus:{
                    if  (ast.insqlpluscmd)
                    {gcurrentsqlstatement.addtokentolist(ast);}
                    else
                    {
                        gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                    break;
                }//case stsqlplus

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if (ast.tokencode == TBaseType.rrw_postgresql_function_delimiter){
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedurePgStartBody;

                        continue;
                    }

                    if (ast.tokencode == TBaseType.rrw_postgresql_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                            }
                        }
                    }

                    if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (!enterDeclare))
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                            )
                    {
                        waitingEnd++;
                        enterDeclare = false;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_declare)
                    ){
                        enterDeclare = true;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not if after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                            waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                            waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                        // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make / a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                        ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        //make ttperiod a sqlplus cmd
                        gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0)
                                && (foundEnd)
                        //        && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())
                        ){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    if (ast.tokencode == TBaseType.sqlpluscmd)
                    {
                        //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                        //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                        int m = flexer.getkeywordvalue(ast.astext);
                        if (m != 0)
                        {ast.tokencode = m;}
                        else
                        {ast.tokencode = TBaseType.ident;}
                    }

                    if (( gst == EFindSqlStateType.ststoredprocedure ) && (ast.tokencode  == TBaseType.cmtdoublehyphen)){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                    break;
                } //ststoredprocedure

                case ststoredprocedurePgStartBody:{
                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_postgresql_function_delimiter){
                        if (gcurrentsqlstatement.sqlstatementtype == sstDoExecuteBlock){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;

                        }else{
                            gst = EFindSqlStateType.ststoredprocedurePgEndBody;
                            continue;
                        }
                    }

                    break;
                }

                case ststoredprocedurePgEndBody:{

                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    else if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);

                    if (ast.tokencode == TBaseType.rrw_postgresql_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                            }
                        }
                    }

                    break;
                }
            } //switch
        }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
                ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql)
                        || (gst == EFindSqlStateType.ststoredprocedure)
                        || (gst == EFindSqlStateType.ststoredprocedurePgEndBody)
                        ||(gst == EFindSqlStateType.sterror)||(isSinglePLBlock)
                ))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return syntaxErrors.size();
    }

    /**
     * Functionality
     * Target Identification:
     * Scans for CREATE PROCEDURE and CREATE FUNCTION statements
     * Verifies the language is SQL (for procedures)
     * Identifies dollar-quoted strings ($$...$$) and single-quoted strings after AS keywords
     * Token Expansion:
     * For each identified string literal:
     * Extracts the content between the quotes
     * Tokenizes the extracted SQL code
     * Verifies it's a valid code block (starts with DECLARE or BEGIN)
     * Replaces the original quoted string with expanded tokens in the source token list
     * Special Transformations:
     * Converts BEGIN; to BEGIN TRANSACTION;
     * Handles BEGIN WORK; similarly
     * Special handling for semicolons after END statements
     * Token Chain Management:
     * Preserves original quotes as space tokens
     * Maintains proper token positioning and chaining
     * Resets the token chain to ensure correct output during toString() operations
     * Implementation Details
     * Processes statements in reverse order to maintain correct token positions
     * Uses TBaseType.getStringInsideLiteral() to extract content between quotes
     * Marks original dollar-quoted tokens as deleted (using tsdeleted)
     * Handles multi-line SQL blocks within stored procedure/function bodies
     */
    void expandDollarString(){
        // snowflake, expand dollar string into token list
        TSourceToken st;
        TCustomSqlStatement sql;
        ArrayList<TSourceToken> dollarTokens = new ArrayList<>();
        boolean isSQLLanguage = true;

        // Iterate all create procedrue and create function, other sql staetment just skipped
        for(int i=0;i<sqlstatements.size();i++){
            sql = sqlstatements.get(i);
            if (!((sql.sqlstatementtype == sstcreateprocedure)||(sql.sqlstatementtype == sstcreatefunction))) continue;
            isSQLLanguage = true;
            for(int j=0;j<sql.sourcetokenlist.size();j++){
                st = sql.sourcetokenlist.get(j);

                if (sql.sqlstatementtype == sstcreateprocedure){
                    if (st.tokencode == TBaseType.rrw_snowflake_language){
                        TSourceToken lang = st.nextSolidToken();
                        if ((lang != null)&&(!lang.toString().equalsIgnoreCase("sql"))){
                            isSQLLanguage = false;
                        }
                    }
                }

                if (!isSQLLanguage) break;

                if (st.tokencode == TBaseType.sconst){
                    if (st.toString().startsWith("$$")){
                        dollarTokens.add(st); // TODO: 可能会有多个 $$ token? 这种情况再哪个场景下会出现？需要找到对应的测试用例和SQL脚本。否则，uncomment next line
                        // break; // only one $$ token per sql, if we found one, then just break this sql and continue for next sql
                    }else if (st.toString().startsWith("'")){
                        // https://docs.snowflake.com/en/sql-reference/sql/create-procedure
                        // string literal delimiter can be $ or '
                        if (st.prevSolidToken().tokencode == TBaseType.rrw_as){
                            dollarTokens.add(st); // TODO: 可能会有多个 $$ token? 这种情况再哪个场景下会出现？需要找到对应的测试用例和SQL脚本。否则，uncomment next line
                            // break; // only one single quote token per sql, if we found one, then just break this sql and continue for next sql
                        }
                    }
                }
            }//check tokens

            for(int m= dollarTokens.size() - 1; m>=0;m--){

                // Token Expansion:
                // For each identified string literal:
                // Extracts the content between the quotes
                // Tokenizes the extracted SQL code
                // Verifies it's a valid code block (starts with DECLARE or BEGIN)
                // Replaces the original quoted string with expanded tokens in the source token list

                st = dollarTokens.get(m);

                TGSqlParser parser = new TGSqlParser(this.dbVendor);
                //parser.sqltext = TBaseType.stringBlock((int)st.lineNo - 1,(int)st.columnNo)+ TBaseType.getStringInsideLiteral(st.toString());
                // 如果在前面补充空行，会导致 toString() 重新拼接时多出空行；
                // 同时，因为不需要进行解析，所以不需要在前面补充空行以获得准确的错误定位? 这个说法是错误的，因为不加需要的空行，会导致定位不准确 #TODO
                parser.sqltext = TBaseType.getStringInsideLiteral(st.toString()); 
                TSourceToken startQuote = new TSourceToken(st.toString().substring(0,1));
                startQuote.tokencode = TBaseType.lexspace; // 设置为 space，可以在解析时忽略该token，同时在 toString() 时会被原样输出
                TSourceToken endQuote = new TSourceToken(st.toString().substring(0,1));
                endQuote.tokencode = TBaseType.lexspace;

                // use getrawsqlstatements() instead of tokenizeSqltext() to get the source token list because
                // some token will be transformed to other token, which will be processed in dosnowflakegetrawsqlstatements()
                parser.getrawsqlstatements();
                //parser.tokenizeSqltext();

                TSourceToken st2;
                boolean isValidBlock = false;
                for(int k=0;k<parser.sourcetokenlist.size();k++){
                    st2 = parser.sourcetokenlist.get(k);
                    if (st2.isnonsolidtoken()) continue;
                    if ((st2.tokencode == TBaseType.rrw_declare)||(st2.tokencode == TBaseType.rrw_begin)){
                        isValidBlock = true;
                    }else{
                        //
                    }
                    break;
                    //System.out.println(st2.lineNo+","+st2.columnNo+":"+st2.toString());
                }

                if (isValidBlock){
                    TSourceToken semiColon = null;
                    st.tokenstatus = ETokenStatus.tsdeleted;
                    int startPosOfThisSQl = sql.getStartToken().posinlist;

                    sql.sourcetokenlist.add((st.posinlist++) - startPosOfThisSQl,startQuote); // 补上开始引号
                    for(int k=0;k<parser.sourcetokenlist.size();k++) {
                        st2 = parser.sourcetokenlist.get(k);
                        if (st2.tokencode == ';'){
                            semiColon = st2;
                            TSourceToken prevSolidToken = st2.prevSolidToken();
                            if ((prevSolidToken != null) && (prevSolidToken.tokencode == TBaseType.rrw_begin)){
                                // begin;  => begin transaction;
                                prevSolidToken.tokencode = TBaseType.rrw_snowflake_begin_transaction;
                            }
                        }
                        if ((st2.tokencode == TBaseType.rrw_snowflake_work)||(st2.tokencode == TBaseType.rrw_snowflake_transaction)){
                            // begin work;  => begin transaction;
                            TSourceToken prevSolidToken = st2.prevSolidToken();
                            if ((prevSolidToken != null) && (prevSolidToken.tokencode == TBaseType.rrw_begin)){
                                // begin;  => begin transaction;
                                prevSolidToken.tokencode = TBaseType.rrw_snowflake_begin_transaction;
                            }
                        }
                        sql.sourcetokenlist.add( (st.posinlist++) - startPosOfThisSQl,st2);
                        //st.posinlist++;
                    }
                    if (semiColon != null){
                        if (semiColon.prevSolidToken().tokencode == TBaseType.rrw_end){
                            // 设置为 space，可以在解析时忽略该token，同时在 toString() 时会被原样输出
                            // semiColon.tokenstatus = ETokenStatus.tsdeleted;
                            semiColon.tokencode = TBaseType.lexspace;
                        }
                    }

                    sql.sourcetokenlist.add( (st.posinlist++) - startPosOfThisSQl,endQuote); // 补上结束引号
                    TBaseType.resetTokenChain(sql.sourcetokenlist,0); // 重新设置token链,确保新插入的token在 toString() 时能被访问到，从而确保 toString() 输出正确结果
                }
            }

            dollarTokens.clear();
        }//statement

    }

int dosnowflakegetrawsqlstatements(){
    int waitingEnd = 0;
    boolean foundEnd = false;

    int waitingEnds[] = new int[stored_procedure_nested_level];
    stored_procedure_type sptype[] = new stored_procedure_type[stored_procedure_nested_level];
    stored_procedure_status procedure_status[] = new stored_procedure_status[stored_procedure_nested_level];
    boolean endBySlashOnly = true;
    int nestedProcedures = 0, nestedParenthesis = 0;
    boolean inDollarBody = false;

    if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
    if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    TSourceToken lcprevsolidtoken = null,ast = null;

    for (int i=0 ; i < sourcetokenlist.size();i++)
      {

          if ( (ast != null ) && (ast.issolidtoken() ))
            lcprevsolidtoken = ast;

        ast = sourcetokenlist.get(i);
        sourcetokenlist.curpos = i;

        if ((ast.tokencode == TBaseType.rrw_right)||(ast.tokencode == TBaseType.rrw_left)){
          TSourceToken stLparen  = ast.searchToken('(',1);
          if (stLparen != null){   //match (
              ast.tokencode = TBaseType.ident;
          }
          TSourceToken stNextToken = ast.nextSolidToken();
          if ((stNextToken != null) && ((stNextToken.tokencode == TBaseType.rrw_join)||(stNextToken.tokencode == TBaseType.rrw_outer))){
            if (ast.tokencode == TBaseType.rrw_left){
                ast.tokencode = TBaseType.rrw_snowflake_left_join;
            }else{
                ast.tokencode = TBaseType.rrw_snowflake_right_join;
            }
          }
        }else if (ast.tokencode == TBaseType.rrw_snowflake_at){
            TSourceToken stLparen  = ast.searchToken('(',1);
            if (stLparen != null){   //match (
                ast.tokencode = TBaseType.rrw_snowflake_at_before_parenthesis;
            }
        }else if (ast.tokencode == TBaseType.rrw_date){
            TSourceToken stLparen  = ast.searchToken('(',1);
            if (stLparen != null){   //date (
                ast.tokencode = TBaseType.rrw_snowflake_date;
            }else {
                stLparen  = ast.searchToken('.',1);
                if (stLparen != null){   //date (
                    ast.tokencode = TBaseType.ident;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_time){
            TSourceToken stLparen  = ast.searchToken('(',1);
            if (stLparen != null){   //date (
                ast.tokencode = TBaseType.rrw_snowflake_time;
            }else {
                stLparen  = ast.searchToken('.',1);
                if (stLparen != null){   //date (
                    ast.tokencode = TBaseType.ident;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_char){
            TSourceToken stLparen  = ast.searchToken('(',1);
            if (stLparen != null){   //date (
                ast.tokencode = TBaseType.rrw_snowflake_char;
            }else {
                stLparen  = ast.searchToken('.',1);
                if (stLparen != null){   //date (
                    ast.tokencode = TBaseType.ident;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_snowflake_window){
            TSourceToken stAs  = ast.searchToken(TBaseType.rrw_as,2);
            if (stAs != null){   //date (
                ast.tokencode = TBaseType.rrw_snowflake_window_as;
            }else {
            }
        }else if ((ast.tokencode == TBaseType.rrw_snowflake_pivot)||(ast.tokencode == TBaseType.rrw_snowflake_unpivot)){
            TSourceToken stLparen  = ast.searchToken('(',1);
            if (stLparen != null){   //pivot (, unpivot

            }else {
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_snowflake_flatten){
            TSourceToken stLeftParens  = ast.searchToken('(',1);
            if (stLeftParens != null){   //flatten (

            }else {
                ast.tokencode = TBaseType.ident; // change it to an identifier, can be used as db object name.
            }
        }else if (ast.tokencode == TBaseType.rrw_snowflake_offset){
            TSourceToken stFrom  = ast.searchToken(TBaseType.rrw_from,-ast.posinlist,TBaseType.rrw_select,true);
            if (stFrom == null){
                // FORM keyword before OFFSET is not found, then offset must be a column name,
                // just like this: SELECT column1 offset FROM table2
                ast.tokencode = TBaseType.ident; // change it to an identifier, can be used as db object name.
            }
        }else if (ast.tokencode == TBaseType.rrw_replace){
            TSourceToken stStar  = ast.prevSolidToken();
            if (stStar.tokencode == '*'){
                ast.tokencode = TBaseType.rrw_snowflake_replace_after_star;
            }
        }else if (ast.tokencode == TBaseType.rrw_snowflake_transaction) {
            TSourceToken stBegin = ast.prevSolidToken();
            if ((stBegin != null) && (stBegin.tokencode == TBaseType.rrw_begin)) {
                stBegin.tokencode = TBaseType.rrw_snowflake_begin_transaction;
            }
        }else if (ast.tokencode == TBaseType.rrw_begin){
            TSourceToken stNext  = ast.nextSolidToken();
            if ((stNext != null) && ((stNext.tokencode == ';')
                        ||(stNext.tokencode == TBaseType.rrw_snowflake_work)||(stNext.tokencode == TBaseType.rrw_snowflake_transaction))
            ){
                ast.tokencode = TBaseType.rrw_snowflake_begin_transaction;
            }
        }else if ((ast.tokencode == TBaseType.rrw_snowflake_top)||(ast.tokencode == TBaseType.rrw_text)||(ast.tokencode == TBaseType.rrw_snowflake_default)){
            TSourceToken stPeriod  = ast.nextSolidToken();
            if ((stPeriod != null) && (stPeriod.tokencode == '.')){
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_snowflake_limit){
            TSourceToken stPrev  = ast.prevSolidToken();
            if ((stPrev != null) && (stPrev.tokencode == ',')){
                ast.tokencode = TBaseType.ident;
            }            
        }else if (ast.tokencode == TBaseType.ident){
            // check whether it is a snowflake parameter name
            // 这个调用可能会有性能问题，因为每个ident都会调用一次
            if (TSnowflakeParameterChecker.isSnowflakeParameter(ast.toString())){
                ast.tokencode = TBaseType.rrw_snowflake_parameter_name;
            }
        }


        switch(gst){
            case sterror:{
                if (ast.tokentype ==  ETokenType.ttsemicolon)
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    gst = EFindSqlStateType.stnormal;
                }
                else
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                break;
            } //sterror

            case stnormal:{
                if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                   || (ast.tokencode  == TBaseType.cmtslashstar)
                   || (ast.tokencode  == TBaseType.lexspace)
                   || (ast.tokencode  == TBaseType.lexnewline)
                   || (ast.tokentype  == ETokenType.ttsemicolon) )
                {
                   if (gcurrentsqlstatement != null)
                   {
                       gcurrentsqlstatement.addtokentolist(ast);
                   }

                   if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                   {
                       if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                       {
                          // ;;;; continuous semicolon,treat it as comment
                           ast.tokentype = ETokenType.ttsimplecomment;
                           ast.tokencode =  TBaseType.cmtdoublehyphen;
                       }
                   }

                   continue;
                }

                if (ast.tokencode == TBaseType.sqlpluscmd )
                {
                    gst = EFindSqlStateType.stsqlplus;
                    gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                    gcurrentsqlstatement.addtokentolist(ast);
                    continue;
                }

                // find a tokentext to start sql or plsql mode
                gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                if (gcurrentsqlstatement != null)
                {
                    if (gcurrentsqlstatement.issnowflakeplsql())
                    {
                        nestedProcedures = 0;
                        gst = EFindSqlStateType.ststoredprocedure;
                        gcurrentsqlstatement.addtokentolist(ast);

                        switch (gcurrentsqlstatement.sqlstatementtype){
                            case sstplsql_createprocedure:
                            case sstcreateprocedure:
                                sptype[nestedProcedures] = stored_procedure_type.procedure;
                                break;
                            case sstplsql_createfunction:
                                sptype[nestedProcedures] = stored_procedure_type.function;
                                break;
                            case sstplsql_createpackage:
                                sptype[nestedProcedures] = stored_procedure_type.package_spec;
                                if (ast.searchToken(TBaseType.rrw_body,5) != null){
                                    sptype[nestedProcedures] = stored_procedure_type.package_body;
                                }
                                break;
                            case sst_plsql_block:
                                sptype[nestedProcedures] = stored_procedure_type.block_with_declare;
                                if (ast.tokencode == TBaseType.rrw_begin) {
                                    sptype[nestedProcedures] = stored_procedure_type.block_with_begin;
                                }
                                break;
                            case sstplsql_createtrigger:
                                sptype[nestedProcedures] = stored_procedure_type.create_trigger;
                                break;
                            case sstoraclecreatelibrary:
                                sptype[nestedProcedures] = stored_procedure_type.create_library;
                                break;
                            case sstplsql_createtype_placeholder:
                                gst = EFindSqlStateType.stsql;
                                break;
                            default:
                                sptype[nestedProcedures] = stored_procedure_type.others;
                                break;
                        }

                        if (sptype[0] == stored_procedure_type.block_with_declare){
                            // sd
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.is_as;
                        }else if (sptype[0] == stored_procedure_type.block_with_begin){
                            // sb
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.body;
                        }else if (sptype[0] == stored_procedure_type.procedure){
                            // ss
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.start;
                        }else if (sptype[0] == stored_procedure_type.function){
                            // ss
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.start;
                        }else if (sptype[0] == stored_procedure_type.package_spec){
                            // ss
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.start;
                        }else if (sptype[0] == stored_procedure_type.package_body){
                            // ss
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.start;
                        }else if (sptype[0] == stored_procedure_type.create_trigger){
                            // ss
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.start;
                            //procedure_status[0] = stored_procedure_status.body;
                        }else if (sptype[0] == stored_procedure_type.create_library){
                            // ss
                            endBySlashOnly = false;
                            procedure_status[0] = stored_procedure_status.bodyend;
                        }else {
                            // so
                            endBySlashOnly = true;
                            procedure_status[0] = stored_procedure_status.bodyend;
                        }
                        //foundEnd = false;
                        if   ((ast.tokencode == TBaseType.rrw_begin)
                                ||(ast.tokencode == TBaseType.rrw_package)
                                //||(ast.tokencode == TBaseType.rrw_procedure)
                                || (ast.searchToken(TBaseType.rrw_package,4) != null)
                        )
                        {
                            //waitingEnd = 1;
                            waitingEnds[nestedProcedures] = 1;
                        }

                    }
                    else
                    {
                        gst = EFindSqlStateType.stsql;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                }else{
                    //error tokentext found

                    this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                            ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                    ast.tokentype = ETokenType.tttokenlizererrortoken;
                    gst = EFindSqlStateType.sterror;

                    gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                    gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                    gcurrentsqlstatement.addtokentolist(ast);

                }

                break;
            } // stnormal

            case stsqlplus:{
                if  (ast.insqlpluscmd)
                {gcurrentsqlstatement.addtokentolist(ast);}
                else
                {
                    gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                    gcurrentsqlstatement.addtokentolist(ast); // so add it here
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                }

                break;
            }//case stsqlplus

            case stsql:{
                if (gcurrentsqlstatement instanceof TRoutine){
                    if (isDollarFunctionDelimiter(ast.tokencode,this.dbVendor)){
                        if (inDollarBody){
                            inDollarBody = false;
                        }else{
                            inDollarBody = true;
                        }
                    }

                    if (inDollarBody) {
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }
                }else if (gcurrentsqlstatement instanceof TCreateTaskStmt){
                    if (ast.tokencode == TBaseType.rrw_as){
                        TSourceToken tmpNext = ast.nextSolidToken();
                        if ((tmpNext != null)&&(tmpNext.tokencode == TBaseType.rrw_begin)){
                            // begin ... end block in create task statement, mantisbt/view.php?id=3531

                            gst = EFindSqlStateType.ststoredprocedure;
                            nestedProcedures = 0;
                            procedure_status[nestedProcedures] = stored_procedure_status.body;
                            waitingEnds[nestedProcedures] = 0;
                            gcurrentsqlstatement.addtokentolist(ast);
                            continue;
                        }
                    }
                }

                if (ast.tokentype == ETokenType.ttsemicolon)
                {
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast);
                    gcurrentsqlstatement.semicolonended = ast;
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    continue;
                }

                if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                {
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast);
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    continue;
                }
                gcurrentsqlstatement.addtokentolist(ast);
                break;
            }//case stsql

            case ststoredprocedure:
            {
                if (procedure_status[nestedProcedures] != stored_procedure_status.bodyend){
                    gcurrentsqlstatement.addtokentolist(ast);
                }

                switch (procedure_status[nestedProcedures]){
                    case start:
                        if ((ast.tokencode == TBaseType.rrw_as)||(ast.tokencode == TBaseType.rrw_is)){
                            // s1
                            if (sptype[nestedProcedures] != stored_procedure_type.create_trigger){
                                if ((sptype[0] == stored_procedure_type.package_spec) && (nestedProcedures > 0)){
                                    //when it's a package specification, only top level accept as/is
                                }else{
                                    procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                                    if (ast.searchToken("language",1) != null){
                                        // if as language is used in create function, then switch state to stored_procedure_status.body directly.
//                                        CREATE OR REPLACE FUNCTION THING.addressparse(p_addressline1 VARCHAR2) RETURN VARCHAR2 AUTHID DEFINER
//                                        as Language JAVA NAME 'AddressParser.parse(java.lang.String) return java.lang.String';
//                                        /
                                        if (nestedProcedures == 0){
                                            //  procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                            gst = EFindSqlStateType.stsql;
                                        }else{
                                            procedure_status[nestedProcedures] = stored_procedure_status.body;
                                            nestedProcedures--;
                                            //if (nestedProcedures > 0){ nestedProcedures--;}
                                        }

                                    }
                                }
                            }
                        }else if(ast.tokencode == TBaseType.rrw_begin){
                            // s4
                            if (sptype[nestedProcedures] == stored_procedure_type.create_trigger) waitingEnds[nestedProcedures]++;

                            if (nestedProcedures > 0) {nestedProcedures--;}
                            procedure_status[nestedProcedures] = stored_procedure_status.body;
                        }else if(ast.tokencode == TBaseType.rrw_end){
                            //s10
                            if((nestedProcedures > 0)&&(waitingEnds[nestedProcedures - 1] == 1)
                                    &&((sptype[nestedProcedures - 1] == stored_procedure_type.package_body)
                                    ||(sptype[nestedProcedures - 1] == stored_procedure_type.package_spec))){
                                nestedProcedures--;
                                procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                            }
                        }else if((ast.tokencode == TBaseType.rrw_procedure)||(ast.tokencode == TBaseType.rrw_function)){
                            //s3
                            if((nestedProcedures > 0)&&(waitingEnds[nestedProcedures] == 0)
                                    &&(procedure_status[nestedProcedures - 1] == stored_procedure_status.is_as)){
                                nestedProcedures--;
                                nestedProcedures++;
                                waitingEnds[nestedProcedures] = 0;
                                procedure_status[nestedProcedures] = stored_procedure_status.start;
                            }
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger) && (ast.tokencode == TBaseType.rrw_declare) )
                        {
                            procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger)&&(ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) )
                        {
                            // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                            ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);

                            //make / a sqlplus cmd
                            gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger) )
                        {
                            if (ast.tokencode == TBaseType.rrw_trigger){
                                TSourceToken compoundSt =  ast.searchToken(TBaseType.rrw_oracle_compound, -1);
                                if (compoundSt != null){
                                    //it's trigger with compound trigger block
                                    procedure_status[nestedProcedures] = stored_procedure_status.body;
                                    waitingEnds[nestedProcedures]++;
                                }
                            }
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.function)&&(ast.tokencode == TBaseType.rrw_teradata_using) )
                        {
                            if ((ast.searchToken("aggregate",-1) != null)||(ast.searchToken("pipelined",-1) != null)){
                                if (nestedProcedures == 0){
                                    //  procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                    gst = EFindSqlStateType.stsql;
                                }else{
                                    procedure_status[nestedProcedures] = stored_procedure_status.body;
                                    nestedProcedures--;
                                }
                            }

                        }else{
                            //other tokens, do nothing
                            if (ast.tokencode == TBaseType.rrw_snowflake_language){
                                // check next token which is the language used by this stored procedure
                                TSourceToken nextSt = ast.nextSolidToken();
                                if (nextSt != null){
                                    if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                        TRoutine p = (TRoutine) gcurrentsqlstatement;
                                        p.setRoutineLanguage(nextSt.toString());
                                        //System.out.println("Find snowflake procedure language: "+p.getRoutineLanguage());
                                        if (p.getRoutineLanguage().toString().equalsIgnoreCase("javascript")){
                                            // procedure 中出现 javascript， 则碰到 semicolon 可能是整个 procedure 语句解释，因此可以设为 stsql 状态
                                            // 但因为 $$body$$ 已经被解析为分离的token，因此需要在 stsql 中忽略这些$$body$$中的token
                                             gst = EFindSqlStateType.stsql;
                                        }
                                    }
                                }
                            }
                        }
                        break;
                    case is_as:
                        if((ast.tokencode == TBaseType.rrw_procedure)||(ast.tokencode == TBaseType.rrw_function)){
                            // s2
                            nestedProcedures++;
                            waitingEnds[nestedProcedures] = 0;
                            procedure_status[nestedProcedures] = stored_procedure_status.start;

                            if (nestedProcedures > stored_procedure_nested_level - 1){
                                gst = EFindSqlStateType.sterror;
                                nestedProcedures--;
                            }

                        }else if(ast.tokencode == TBaseType.rrw_begin){
                            // s5
                            if ((nestedProcedures == 0)&&
                                    ((sptype[nestedProcedures]==stored_procedure_type.package_body)
                                            ||(sptype[nestedProcedures]==stored_procedure_type.package_spec))){
                                //top level package or package body's BEGIN keyword already count,
                                // so don't increase waitingEnds[nestedProcedures] here

                            }else {
                                waitingEnds[nestedProcedures]++;
                            }
                            procedure_status[nestedProcedures] = stored_procedure_status.body;
                        }else if(ast.tokencode == TBaseType.rrw_end){
                            // s6
                            if ((nestedProcedures == 0)&&(waitingEnds[nestedProcedures] == 1)&&
                                    ((sptype[nestedProcedures]==stored_procedure_type.package_body)||(sptype[nestedProcedures]==stored_procedure_type.package_spec))){
                                procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                waitingEnds[nestedProcedures]--;
                            }else{
                                waitingEnds[nestedProcedures]--;
                            }
                        }else if(ast.tokencode == TBaseType.rrw_case)
                        {
//                            if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                //this is not case after END
//                             waitingEnds[nestedProcedures]++;
//                            }
                            if (ast.searchToken(';',1) == null){
                                //this is not case before ;
                                waitingEnds[nestedProcedures]++;
                            }
                        }
                        else {
                            //other tokens, do nothing
                        }
                        break;
                    case body:
                        if ((ast.tokencode == TBaseType.rrw_begin))
                        {
                            waitingEnds[nestedProcedures]++;
                        }
                        else if (ast.tokencode == TBaseType.rrw_if)
                        {
                             if (ast.searchToken(TBaseType.rrw_snowflake_exists,1) != null){
                                 //drop table if exists SANDBOX.ANALYSIS_CONTENT.TABLEAU_DATES;
                                 // don't need END for the above if exists clause
                            }
                            else if (ast.searchToken(';',2) == null){
                                //this is not if before ;

                                // 2015-02-27, change 1 to 2 make it able to detect label name after case
                                // like this: END CASE l1;
                                waitingEnds[nestedProcedures]++;
                            }
                        }
                        else if (ast.tokencode == TBaseType.rrw_for)
                        {
                            if (ast.searchToken(';',2) == null){
                                //this is not for before ;

                                // 2015-02-27, change 1 to 2 make it able to detect label name after case
                                // like this: END CASE l1;
                                waitingEnds[nestedProcedures]++;
                            }
                        }
                        else if(ast.tokencode == TBaseType.rrw_case)
                        {
//                            if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                //this is not case after END
//                             waitingEnds[nestedProcedures]++;
//                            }
                            if (ast.searchToken(';',2) == null){
                                //this is not case before ;
                                if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                                    waitingEnds[nestedProcedures]++;
                                }
                            }
                        } else if(ast.tokencode == TBaseType.rrw_loop)
                        {
                            if (!((ast.searchToken(TBaseType.rrw_end,-1) != null)
                                    &&(ast.searchToken(';',2) != null))){
                                // exclude loop like this:
                                // end loop [labelname];
                                waitingEnds[nestedProcedures]++;
                            }

//                            if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                //this is not loop after END
//                             waitingEnds[nestedProcedures]++;
////                            }
////                            if (ast.searchToken(';',2) == null){
////                                //this is no loop before ;
////                             waitingEnds[nestedProcedures]++;
//                            } else if (ast.searchToken(TBaseType.rrw_null,1) != null){
//                                // mantis bug tracking system:   #65
//                                waitingEnds[nestedProcedures]++;
//                            }
                        }else if (ast.tokencode == TBaseType.rrw_end){
                            //foundEnd = true;
                            waitingEnds[nestedProcedures]--;
                            //if (waitingEnd < 0) { waitingEnd = 0;}
                            if (waitingEnds[nestedProcedures] == 0){
                                if (nestedProcedures == 0){
                                    // s7
                                    procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                }else {
                                    // s71
                                    nestedProcedures--;
                                    procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                                }
                            }
                        }else  if ((waitingEnds[nestedProcedures] == 0)&&(ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                        {
                            //sql ref: c:\prg\gsqlparser\Test\TestCases\oracle\createtrigger.sql, line 53
                            ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);

                            //make / a sqlplus cmd
                            gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                        break;
                    case bodyend:
                        if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                        {
                            // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                            ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);

                            //make / a sqlplus cmd
                            gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                        {    // single dot at a seperate line
                            ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);

                            //make ttperiod a sqlplus cmd
                            gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_package,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_procedure,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_function,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_create,1) != null)&&(ast.searchToken(TBaseType.rrw_package,4) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_create,1) != null)&&(ast.searchToken(TBaseType.rrw_library,4) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_alter,1) != null)&&(ast.searchToken(TBaseType.rrw_trigger,2) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_select,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_commit,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_grant,1) != null)&&
                                (ast.searchToken(TBaseType.rrw_execute,2) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else if ((gcurrentsqlstatement instanceof TCreateTaskStmt)&&(ast.tokencode == ';')){
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                        else {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                        break;
                    case end:
                        break;
                    default:
                        break;
                }


                if (ast.tokencode == TBaseType.sqlpluscmd)
                {
                    //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                    //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                    int m = flexer.getkeywordvalue(ast.astext);
                    if (m != 0)
                    {ast.tokencode = m;}
                    else if (ast.tokentype == ETokenType.ttslash){
                        ast.tokencode = '/';
                    }
                    else
                    {ast.tokencode = TBaseType.ident;}
                }

                final int wrapped_keyword_max_pos = 20;
                if ((ast.tokencode == TBaseType.rrw_wrapped)&&(ast.posinlist - gcurrentsqlstatement.sourcetokenlist.get(0).posinlist  < wrapped_keyword_max_pos)){
                    if (gcurrentsqlstatement instanceof TCommonStoredProcedureSqlStatement){
                        ((TCommonStoredProcedureSqlStatement)gcurrentsqlstatement).setWrapped(true);
                    }

                    if (gcurrentsqlstatement instanceof TPlsqlCreatePackage){
                        if (ast.prevSolidToken() != null){
                            ((TPlsqlCreatePackage) gcurrentsqlstatement).setPackageName(fparser.getNf().createObjectNameWithPart(ast.prevSolidToken()));
                        }
                    }
                }

                break;
            } //ststoredprocedure
        } //switch
      }//for

    //last statement
    if ((gcurrentsqlstatement != null) &&
        ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                    (gst == EFindSqlStateType.sterror)))
    {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
    }

  return syntaxErrors.size();
}

int dogreenplumgetrawsqlstatements(){
        int waitingEnd = 0;
        boolean foundEnd = false,enterDeclare=false;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;
        TSourceToken dollarStringToken = null;

        for (int i=0 ; i < sourcetokenlist.size();i++)
          {

              if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

              if (ast.tokencode == TBaseType.rrw_date) {
                  TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                  if (st1 != null) {
                      if (st1.tokencode == '(') {
                          ast.tokencode = TBaseType.rrw_greenplum_DATE_FUNCTION;
                      }
                  }
              }
              else if (ast.tokencode == TBaseType.rrw_greenplum_POSITION) {
                  TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                  if (st1 != null) {
                      if (st1.tokencode == '(') {
                          ast.tokencode = TBaseType.rrw_greenplum_POSITION_FUNCTION;
                      }
                  }
              }
              else if (ast.tokencode == TBaseType.rrw_greenplum_filter) {
                  TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                  if (st1 != null) {
                      if (st1.tokencode == '(') {

                      }else{
                          ast.tokencode = TBaseType.ident;
                      }
                  }
              }
              else if (ast.tokencode == TBaseType.rrw_values) {
                  TSourceToken stParen =  ast.searchToken('(',1);
                  if (stParen != null){
                      TSourceToken stInsert  = ast.searchToken(TBaseType.rrw_insert,-ast.posinlist);
                      if (stInsert != null){
                          TSourceToken stSemiColon  = ast.searchToken(';',-ast.posinlist);
                          if ((stSemiColon != null)&&(stSemiColon.posinlist > stInsert.posinlist)){
//                            INSERT INTO test values (16,1), (8,2), (4,4), (2,0), (97, 16);
//                            VALUES (1);
                              // don't treat values(1) as insert values

                          }else{
                              TSourceToken stFrom  = ast.searchToken(TBaseType.rrw_from,-ast.posinlist);
                              if (stFrom != null){
                                  // don't treat values after from keyword as a insert values

                                  // insert into inserttest values(10, 20, '40'), (-1, 2, DEFAULT),  ((select 2), (select i from (values(3) ) as foo (i)), 'values are fun!');

                              }else{
                                  ast.tokencode = TBaseType.rrw_greenplum_values_insert;
                              }

                          }

                      }
                  }
              }

              switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                       || (ast.tokencode  == TBaseType.cmtslashstar)
                       || (ast.tokencode  == TBaseType.lexspace)
                       || (ast.tokencode  == TBaseType.lexnewline)
                       || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                       if (gcurrentsqlstatement != null)
                       {
                           gcurrentsqlstatement.addtokentolist(ast);
                       }

                       if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                       {
                           if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                           {
                              // ;;;; continuous semicolon,treat it as comment
                               ast.tokentype = ETokenType.ttsimplecomment;
                               ast.tokencode =  TBaseType.cmtdoublehyphen;
                           }
                       }

                       continue;
                    }

                    if (ast.tokencode == '\\' )
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TSlashCommand(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                        if (gcurrentsqlstatement.isgreeplumplsql())
                        {
                              gst = EFindSqlStateType.ststoredprocedure;
                              gcurrentsqlstatement.addtokentolist(ast);
                              foundEnd = false;
                              if   ((ast.tokencode == TBaseType.rrw_begin)
                                ||(ast.tokencode == TBaseType.rrw_package)
                                || (ast.searchToken(TBaseType.rrw_package,4) != null)
                                      )
                              {
                                  waitingEnd = 1;
                              }else if (ast.tokencode == TBaseType.rrw_declare){
                                  enterDeclare = true;
                              }
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsqlplus:{
                    if (ast.tokencode  == TBaseType.lexnewline){
                        gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }if (ast.tokencode  == '\\'){
                        TSourceToken nextst = ast.searchToken('\\',1);
                        if (nextst!=null){
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }else {
                            gst = EFindSqlStateType.stsqlplus;
                            gcurrentsqlstatement = new TSlashCommand(dbVendor);
                            gcurrentsqlstatement.addtokentolist(ast);
                            continue;
                        }
                    }else {
                        {gcurrentsqlstatement.addtokentolist(ast);}
                    }


                    break;
                }//case greenplum meta command

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

                case ststoredprocedure:{
                    if (ast.tokencode == TBaseType.rrw_greenplum_function_delimiter){
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedurePgStartBody;
                        dollarStringToken = ast;
                        continue;
                    }

                    if (ast.tokencode == TBaseType.rrw_greenplum_language){
                        // check next token which is the language used by this stored procedure
                        TSourceToken nextSt = ast.nextSolidToken();
                        if (nextSt != null){
                            if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                TRoutine p = (TRoutine) gcurrentsqlstatement;
                                p.setRoutineLanguage(nextSt.toString());
                            }
                        }
                    }

                    if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (!enterDeclare))
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }


                    if   ((ast.tokencode == TBaseType.rrw_begin)
                            //||(ast.tokencode == TBaseType.rrw_between)
                            )
                    {
                        waitingEnd++;
                        enterDeclare = false;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_declare)
                    ){
                        enterDeclare = true;
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_if)
                            ){
                         if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                             //this is not if after END
                             if(!( (ast.searchToken(TBaseType.rrw_greenplum_exits,1) != null)
                                 ||(ast.searchToken(TBaseType.rrw_greenplum_exits,2) != null)
                             )) // if not exists, if exists is not a real if statement, so skip those
                             {
                                 waitingEnd++;
                             }
                         }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_case)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not case after END
                         waitingEnd++;
                        }
                    }
                    else if   (
                            (ast.tokencode == TBaseType.rrw_loop)
                            ){
                        if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                            //this is not loop after END
                         waitingEnd++;
                        }
                    }
                    else if (ast.tokencode == TBaseType.rrw_end){
                        foundEnd = true;
                        waitingEnd--;
                        if (waitingEnd < 0) { waitingEnd = 0;}
                    }

                    if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                    {
                      // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                       ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                       gst = EFindSqlStateType.stnormal;
                       doongetrawsqlstatementevent(gcurrentsqlstatement);

                       //make / a sqlplus cmd
                       gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                       gcurrentsqlstatement.addtokentolist(ast);
                       doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }
                    else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                    {    // single dot at a seperate line
                       ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                       gst = EFindSqlStateType.stnormal;
                       doongetrawsqlstatementevent(gcurrentsqlstatement);

                       //make ttperiod a sqlplus cmd
                       gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                       gcurrentsqlstatement.addtokentolist(ast);
                       doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }else  if ((ast.tokencode == TBaseType.rrw_declare)&&(waitingEnd == 0)){
                        i--;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                     else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (foundEnd) && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                    }

                     if (ast.tokencode == TBaseType.sqlpluscmd)
                     {
                         //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                         //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                         int m = flexer.getkeywordvalue(ast.astext);
                         if (m != 0)
                         {ast.tokencode = m;}
                         else
                         {ast.tokencode = TBaseType.ident;}
                     }

                    break;
                } //ststoredprocedure

                  case ststoredprocedurePgStartBody:{
                      gcurrentsqlstatement.addtokentolist(ast);

                      if (ast.tokencode == TBaseType.rrw_greenplum_function_delimiter){
                          if (dollarStringToken.toString().equalsIgnoreCase(ast.toString())){
                              // must match the $$ token
                              gst = EFindSqlStateType.ststoredprocedurePgEndBody;
                              continue;
                          }
                      }

                      break;
                  }

                  case ststoredprocedurePgEndBody:{

                      if (ast.tokentype == ETokenType.ttsemicolon)
                      {
                          gst = EFindSqlStateType.stnormal;
                          gcurrentsqlstatement.addtokentolist(ast);
                          gcurrentsqlstatement.semicolonended = ast;
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          continue;
                      }
                      else if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                          if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                              gst = EFindSqlStateType.stnormal;
                              doongetrawsqlstatementevent(gcurrentsqlstatement);
                              continue;
                          }
                      }

                      gcurrentsqlstatement.addtokentolist(ast);

                      if (ast.tokencode == TBaseType.rrw_greenplum_language){
                          // check next token which is the language used by this stored procedure
                          TSourceToken nextSt = ast.nextSolidToken();
                          if (nextSt != null){
                              if (gcurrentsqlstatement instanceof TRoutine){  // can be TCreateProcedureStmt or TCreateFunctionStmt
                                  TRoutine p = (TRoutine) gcurrentsqlstatement;
                                  p.setRoutineLanguage(nextSt.toString());
                              }
                          }
                      }

                      break;
                  }

            } //switch
          }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
            ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql)
                    || (gst == EFindSqlStateType.ststoredprocedure)|| (gst == EFindSqlStateType.ststoredprocedurePgEndBody)
                    ||(gst == EFindSqlStateType.sterror)))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

      return syntaxErrors.size();
    }

private boolean isTypeCastToken(TSourceToken ast){
    boolean istypecasetoken = false;
    TSourceToken st = ast.searchToken('(',1);
    if (st != null){
       TSourceToken  nst = st.searchToken(TBaseType.iconst,1);
        istypecasetoken = (nst == null);
    }

    return istypecasetoken;
}

int donetezzagetrawsqlstatements(){
    int waitingEnd = 0;
    boolean foundEnd = false;

    if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
    if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    TSourceToken lcprevsolidtoken = null,ast = null;

    for (int i=0 ; i < sourcetokenlist.size();i++)
      {

          if ( (ast != null ) && (ast.issolidtoken() ))
            lcprevsolidtoken = ast;

        ast = sourcetokenlist.get(i);
        sourcetokenlist.curpos = i;

        // change token code if necessary
        if (ast.tokencode == TBaseType.rrw_int){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_int_cast;
        }else  if (ast.tokencode == TBaseType.rrw_integer){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_integer_cast;
        }else  if (ast.tokencode == TBaseType.rrw_smallint){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_smallint_cast;
        }else  if (ast.tokencode == TBaseType.rrw_bigint){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_bigint_cast;
        }else  if (ast.tokencode == TBaseType.rrw_real){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_real_cast;
        }else  if (ast.tokencode == TBaseType.rrw_float){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_float_cast;
        }else  if (ast.tokencode == TBaseType.rrw_numeric){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_numeric_cast;
//        }else  if (ast.tokencode == TBaseType.rrw_boolean){
//            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_boolean_cast;
        }else  if (ast.tokencode == TBaseType.rrw_bit){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_bit_cast;
        }else  if (ast.tokencode == TBaseType.rrw_char){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_char_cast;
        }else  if (ast.tokencode == TBaseType.rrw_nchar){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_nchar_cast;
        }else  if (ast.tokencode == TBaseType.rrw_varchar){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_varchar_cast;
        }else  if (ast.tokencode == TBaseType.rrw_character){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_character_cast;
        }else  if (ast.tokencode == TBaseType.rrw_date){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_date_cast;
        }else  if (ast.tokencode == TBaseType.rrw_time){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_time_cast;
        }else  if (ast.tokencode == TBaseType.rrw_timestamp){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_timestamp_cast;
        }else  if (ast.tokencode == TBaseType.rrw_interval){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_interval_cast;
        }else  if (ast.tokencode == TBaseType.rrw_decimal){
            if (isTypeCastToken(ast)) ast.tokencode = TBaseType.rrw_decimal_cast;
        }

        switch(gst){
            case sterror:{
                if (ast.tokentype ==  ETokenType.ttsemicolon)
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    gst = EFindSqlStateType.stnormal;
                }
                else
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                break;
            } //sterror

            case stnormal:{
                if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                   || (ast.tokencode  == TBaseType.cmtslashstar)
                   || (ast.tokencode  == TBaseType.lexspace)
                   || (ast.tokencode  == TBaseType.lexnewline)
                   || (ast.tokentype  == ETokenType.ttsemicolon) )
                {
                   if (gcurrentsqlstatement != null)
                   {
                       gcurrentsqlstatement.addtokentolist(ast);
                   }

                   if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                   {
                       if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                       {
                          // ;;;; continuous semicolon,treat it as comment
                           ast.tokentype = ETokenType.ttsimplecomment;
                           ast.tokencode =  TBaseType.cmtdoublehyphen;
                       }
                   }

                   continue;
                }

                if (ast.tokencode == TBaseType.sqlpluscmd )
                {
                    gst = EFindSqlStateType.stsqlplus;
                    gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                    gcurrentsqlstatement.addtokentolist(ast);
                    continue;
                }

                // find a tokentext to start sql or plsql mode
                gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                if (gcurrentsqlstatement != null)
                {
                    if (gcurrentsqlstatement.isnzplsql())
                    {
                          gst = EFindSqlStateType.ststoredprocedure;
                          gcurrentsqlstatement.addtokentolist(ast);
                          foundEnd = false;
                          if   ((ast.tokencode == TBaseType.rrw_begin)
                            ||(ast.tokencode == TBaseType.rrw_package)
                            || (ast.searchToken(TBaseType.rrw_package,4) != null)
                                  )
                          { waitingEnd = 1;}
                    }
                    else
                    {
                        gst = EFindSqlStateType.stsql;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                }else{
                    //error tokentext found

                    this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                            ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                    ast.tokentype = ETokenType.tttokenlizererrortoken;
                    gst = EFindSqlStateType.sterror;

                    gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                    gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                    gcurrentsqlstatement.addtokentolist(ast);

                }

                break;
            } // stnormal

            case stsqlplus:{
                if  (ast.insqlpluscmd)
                {gcurrentsqlstatement.addtokentolist(ast);}
                else
                {
                    gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                    gcurrentsqlstatement.addtokentolist(ast); // so add it here
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                }

                break;
            }//case stsqlplus

            case stsql:{
                if (ast.tokentype == ETokenType.ttsemicolon)
                {
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast);
                    gcurrentsqlstatement.semicolonended = ast;
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    continue;
                }

                if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probaly is / cmd
                {
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast);
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    continue;
                }
                gcurrentsqlstatement.addtokentolist(ast);
                break;
            }//case stsql

            case ststoredprocedure:{
                if   ((ast.tokencode == TBaseType.rrw_begin)
                        //||(ast.tokencode == TBaseType.rrw_between)
                        )
                {
                    waitingEnd++;
                }
                else if   (
                        (ast.tokencode == TBaseType.rrw_if)
                        ){
                     if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                         //this is not if after END
                      waitingEnd++;
                     }
                }
                else if   (
                        (ast.tokencode == TBaseType.rrw_case)
                        ){
                    if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                        //this is not case after END
                     waitingEnd++;
                    }
                }
                else if   (
                        (ast.tokencode == TBaseType.rrw_loop)
                        ){
                    if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                        //this is not loop after END
                     waitingEnd++;
                    }
                }
                else if (ast.tokencode == TBaseType.rrw_end){
                    foundEnd = true;
                    waitingEnd--;
                    if (waitingEnd < 0) { waitingEnd = 0;}
                }

                if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                {
                  // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                   ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                   gst = EFindSqlStateType.stnormal;
                   doongetrawsqlstatementevent(gcurrentsqlstatement);

                   //make / a sqlplus cmd
                   gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                   gcurrentsqlstatement.addtokentolist(ast);
                   doongetrawsqlstatementevent(gcurrentsqlstatement);
                }
                else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                {    // single dot at a seperate line
                   ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                   gst = EFindSqlStateType.stnormal;
                   doongetrawsqlstatementevent(gcurrentsqlstatement);

                   //make ttperiod a sqlplus cmd
                   gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                   gcurrentsqlstatement.addtokentolist(ast);
                   doongetrawsqlstatementevent(gcurrentsqlstatement);
                }
                 else
                {
                    gcurrentsqlstatement.addtokentolist(ast);
                    if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (foundEnd) && (gcurrentsqlstatement.OracleStatementCanBeSeparatedByBeginEndPair())){
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }else if ((ast.searchToken(TBaseType.rrw_create,1) != null)&&(ast.searchToken(TBaseType.rrw_procedure,4) != null)&&(waitingEnd == 0)){
                        gst = EFindSqlStateType.stnormal;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

            }

                break;
            } //ststoredprocedure
        } //switch
      }//for

    //last statement
    if ((gcurrentsqlstatement != null) &&
        ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                    (gst == EFindSqlStateType.sterror)))
    {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
    }

  return syntaxErrors.size();
}

int dooraclegetrawsqlstatements(){
    int waitingEnds[] = new int[stored_procedure_nested_level];
    stored_procedure_type sptype[] = new stored_procedure_type[stored_procedure_nested_level];
    stored_procedure_status procedure_status[] = new stored_procedure_status[stored_procedure_nested_level];
    boolean endBySlashOnly = true;
    int nestedProcedures = 0, nestedParenthesis = 0;



    if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
    if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    TSourceToken lcprevsolidtoken = null,ast = null;

    for (int i=0 ; i < sourcetokenlist.size();i++)
      {

          if ( (ast != null ) && (ast.issolidtoken() ))
            lcprevsolidtoken = ast;

        ast = sourcetokenlist.get(i);
        sourcetokenlist.curpos = i;

        if (ast.tokencode == TBaseType.rrw_return){
                    TSourceToken stMatch  = ast.searchToken(TBaseType.rrw_where,1);
                    if (stMatch != null){   //match full
                        ast.tokencode = TBaseType.ident;
                    }
        }else if (ast.tokencode == TBaseType.rrw_value_oracle){
          TSourceToken stBy  = ast.searchToken(TBaseType.rrw_by,-1);
          if (stBy == null){
             // keep value as keyword, but it is in rule: unreserved_keyword_can_be_function
              // so value can be used as column ,table name.

          }else{
              ast.tokencode = TBaseType.rrw_value_after_by;
          }
        }else if (ast.tokencode == TBaseType.rrw_new_oracle){
            TSourceToken stRightParen  = ast.searchToken(')',-1);
            if (stRightParen != null){
                ast.tokencode = TBaseType.ident;
            }
            TSourceToken stDot  = ast.searchToken('.',1);
            if (stDot != null){
                ast.tokencode = TBaseType.ident;
            }
//            TSourceToken stNext= ast.searchToken("(",3);
//            if (stNext != null){
//                ast.tokencode = TBaseType.rrw_oracle_new_constructor;
//            }

            TSourceToken stNext= ast.searchTokenAfterObjectName();
            stDot  = ast.searchToken('.',1);
            if ((stDot == null)&&(stNext != null) && (stNext.tokencode == '(')){
                ast.tokencode = TBaseType.rrw_oracle_new_constructor;
            }

        }else if (ast.tokencode == TBaseType.rrw_chr_oracle){
            TSourceToken stLeftParen= ast.searchToken('(',1);
            if (stLeftParen == null){
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_log_oracle){
            TSourceToken stNext= ast.searchToken(TBaseType.rrw_errors_oracle,1);
            TSourceToken stPrev= ast.searchToken(TBaseType.rrw_view,-1);
            if (stPrev == null){
                stPrev= ast.searchToken(TBaseType.rrw_oracle_supplemental,-1);
            }
            if ((stNext == null)&&(stPrev == null)){
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_delete){
            TSourceToken stPrev= ast.searchToken('.',-1);
            if (stPrev != null){
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_partition){
            TSourceToken stPrev= ast.searchToken(TBaseType.rrw_add,-1);
            if (stPrev != null){
                stPrev.tokencode = TBaseType.rrw_add_p;
            }
        }else if (ast.tokencode == TBaseType.rrw_oracle_column){
            TSourceToken stPrev= ast.searchToken(TBaseType.rrw_oracle_modify,-1);
            if (stPrev != null){
                ast.tokencode = TBaseType.rrw_oracle_column_after_modify;
            }
        }else if (ast.tokencode == TBaseType.rrw_oracle_apply){
            TSourceToken stPrev= ast.searchToken(TBaseType.rrw_outer,-1);
            if (stPrev != null){
                stPrev.tokencode = TBaseType.ORACLE_OUTER2;
            }
        }else if (ast.tokencode == TBaseType.rrw_oracle_subpartition){
            TSourceToken stNext= ast.searchToken("(",2);
            if (stNext != null){

                TSourceToken st1 = ast.nextSolidToken();
                if (st1.toString().equalsIgnoreCase("template")){
                    // RW_SUBPARTITION RW_TEMPLATE '(' dummy_node ')'
                    // don't change, keep as RW_SUBPARTITION
                }else{
                    ast.tokencode = TBaseType.rrw_oracle_subpartition_tablesample;
                }

            }
        }else if (ast.tokencode == TBaseType.rrw_primary){
            TSourceToken stNext= ast.searchToken("key",1);
            if (stNext == null){ // if this is not primary key, then, treat "primary" as an identifier.
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_oracle_offset){
            TSourceToken stNext= ast.searchToken(TBaseType.rrw_oracle_row,2);
            if (stNext == null){
                stNext= ast.searchToken(TBaseType.rrw_oracle_rows,2);
            }
            if (stNext != null){
                ast.tokencode = TBaseType.rrw_oracle_offset_row;
            }
        }else if (ast.tokencode == TBaseType.rrw_translate){
            TSourceToken stNext= ast.searchToken("(",2);
            if (stNext == null){
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_constraint){
            TSourceToken stNext= ast.nextSolidToken();
            if (stNext == null){
                ast.tokencode = TBaseType.ident;
            }else{
                if (stNext.tokencode != TBaseType.ident){
                    // constraint name, there must be a name after keyword constraint, otherwise, constraint is an identifier
                    ast.tokencode = TBaseType.ident;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_oracle_without) {
            TSourceToken stNext = ast.searchToken(TBaseType.rrw_oracle_count, 1);
            if (stNext != null) {
                ast.tokencode = TBaseType.rrw_oracle_without_before_count;
            }
        }else if (ast.tokencode == TBaseType.rrw_bulk) {
            TSourceToken stNext = ast.searchToken(TBaseType.rrw_oracle_collect, 1);
            if (stNext == null) {
                // treat all bulk keywords as identifier if not followed by collect.
                // bulk collect
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_oracle_model){
            TSourceToken stNext = ast.nextSolidToken();
            if (stNext != null){
                switch (stNext.toString().toUpperCase()) {
                    case "RETURN":      // RETURN [UPDATED|ALL] ROWS
                    case "REFERENCE":   // REFERENCE model_name...
                    case "IGNORE":      // IGNORE NAV
                    case "KEEP":        // KEEP NAV
                    case "UNIQUE":      // UNIQUE [DIMENSION|SINGLE REFERENCE]
                    case "PARTITION":   // PARTITION BY
                    case "DIMENSION":   // DIMENSION BY
                    case "MEASURES":    // MEASURES (cols)
                    case "RULES":       // RULES...
                        ast.tokencode = TBaseType.rrw_oracle_model_in_model_clause;
                        break;
                    default:
                       ;
                }
            }
        }

        switch(gst){
            case sterror:{
                if (ast.tokentype ==  ETokenType.ttsemicolon)
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    gst = EFindSqlStateType.stnormal;
                }
                else
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                break;
            } //sterror

            case stnormal:{
                if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                   || (ast.tokencode  == TBaseType.cmtslashstar)
                   || (ast.tokencode  == TBaseType.lexspace)
                   || (ast.tokencode  == TBaseType.lexnewline)
                   || (ast.tokentype  == ETokenType.ttsemicolon) )
                {
                   if (gcurrentsqlstatement != null)
                   {
                       gcurrentsqlstatement.addtokentolist(ast);
                   }

                   if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                   {
                       if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                       {
                          // ;;;; continuous semicolon,treat it as comment
                           ast.tokentype = ETokenType.ttsimplecomment;
                           ast.tokencode =  TBaseType.cmtdoublehyphen;
                       }
                   }

                   continue;
                }

                if (ast.tokencode == TBaseType.sqlpluscmd )
                {

                    gst = EFindSqlStateType.stsqlplus;
                    gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                    gcurrentsqlstatement.addtokentolist(ast);
                    continue;
                }

                // find a tokentext to start sql or plsql mode
                gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                if (gcurrentsqlstatement != null)
                {
                    if (gcurrentsqlstatement.isoracleplsql())
                    {
                          nestedProcedures = 0;
                          gst = EFindSqlStateType.ststoredprocedure;
                          gcurrentsqlstatement.addtokentolist(ast);

                          switch (gcurrentsqlstatement.sqlstatementtype){
                              case sstplsql_createprocedure:
                                  sptype[nestedProcedures] = stored_procedure_type.procedure;
                                  break;
                              case sstplsql_createfunction:
                                  sptype[nestedProcedures] = stored_procedure_type.function;
                                  break;
                              case sstplsql_createpackage:
                                  sptype[nestedProcedures] = stored_procedure_type.package_spec;
                                  if (ast.searchToken(TBaseType.rrw_body,5) != null){
                                      sptype[nestedProcedures] = stored_procedure_type.package_body;
                                  }
                                  break;
                              case sst_plsql_block:
                                  sptype[nestedProcedures] = stored_procedure_type.block_with_declare;
                                  if (ast.tokencode == TBaseType.rrw_begin) {
                                      sptype[nestedProcedures] = stored_procedure_type.block_with_begin;
                                  }
                                  break;
                              case sstplsql_createtrigger:
                                  sptype[nestedProcedures] = stored_procedure_type.create_trigger;
                                  break;
                              case sstoraclecreatelibrary:
                                  sptype[nestedProcedures] = stored_procedure_type.create_library;
                                  break;
                              case sstplsql_createtype_placeholder:
                                  gst = EFindSqlStateType.stsql;
                                  break;
                              default:
                                  sptype[nestedProcedures] = stored_procedure_type.others;
                                  break;
                          }

                          if (sptype[0] == stored_procedure_type.block_with_declare){
                              // sd
                             endBySlashOnly = false;
                             procedure_status[0] = stored_procedure_status.is_as;
                          }else if (sptype[0] == stored_procedure_type.block_with_begin){
                              // sb
                              endBySlashOnly = false;
                              procedure_status[0] = stored_procedure_status.body;
                          }else if (sptype[0] == stored_procedure_type.procedure){
                              // ss
                              endBySlashOnly = false;
                              procedure_status[0] = stored_procedure_status.start;
                          }else if (sptype[0] == stored_procedure_type.function){
                              // ss
                              endBySlashOnly = false;
                              procedure_status[0] = stored_procedure_status.start;
                          }else if (sptype[0] == stored_procedure_type.package_spec){
                              // ss
                              endBySlashOnly = false;
                              procedure_status[0] = stored_procedure_status.start;
                          }else if (sptype[0] == stored_procedure_type.package_body){
                              // ss
                              endBySlashOnly = false;
                              procedure_status[0] = stored_procedure_status.start;
                          }else if (sptype[0] == stored_procedure_type.create_trigger){
                              // ss
                              endBySlashOnly = false;
                              procedure_status[0] = stored_procedure_status.start;
                              //procedure_status[0] = stored_procedure_status.body;
                          }else if (sptype[0] == stored_procedure_type.create_library){
                              // ss
                              endBySlashOnly = false;
                              procedure_status[0] = stored_procedure_status.bodyend;
                          }else {
                              // so
                              endBySlashOnly = true;
                              procedure_status[0] = stored_procedure_status.bodyend;
                          }
                          //foundEnd = false;
                          if   ((ast.tokencode == TBaseType.rrw_begin)
                            ||(ast.tokencode == TBaseType.rrw_package)
                            //||(ast.tokencode == TBaseType.rrw_procedure)
                            || (ast.searchToken(TBaseType.rrw_package,4) != null)
                                  )
                          {
                              //waitingEnd = 1;
                              waitingEnds[nestedProcedures] = 1;
                          }
                    }
                    else
                    {
                        gst = EFindSqlStateType.stsql;
                        gcurrentsqlstatement.addtokentolist(ast);
                        nestedParenthesis = 0;
                    }
                }else{
                    //error tokentext found

                    this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                            ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                    ast.tokentype = ETokenType.tttokenlizererrortoken;
                    gst = EFindSqlStateType.sterror;

                    gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                    gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                    gcurrentsqlstatement.addtokentolist(ast);

                }

                break;
            } // stnormal

            case stsqlplus:{
                if  (ast.insqlpluscmd)
                {gcurrentsqlstatement.addtokentolist(ast);}
                else
                {
                    gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                    gcurrentsqlstatement.addtokentolist(ast); // so add it here
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                }

                break;
            }//case stsqlplus

            case stsql:{
                if (ast.tokentype == ETokenType.ttsemicolon)
                {
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast);
                    gcurrentsqlstatement.semicolonended = ast;
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    continue;
                }

                if  (sourcetokenlist.sqlplusaftercurtoken() ) //most probably is / cmd
                {
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast);
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    continue;
                }

                if (ast.tokencode == '(') nestedParenthesis++;
                if (ast.tokencode == ')')
                {
                    nestedParenthesis--;
                    if (nestedParenthesis < 0) nestedParenthesis = 0;
                }
                Boolean findNewStmt = false;
                TCustomSqlStatement lcStmt = null;
                if ((nestedParenthesis == 0) && (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatetable))
                {
                    lcStmt = sqlcmds.issql(ast, dbVendor, gst, gcurrentsqlstatement);
                    if (lcStmt != null)
                    {
                        findNewStmt = true;
                        if (lcStmt.sqlstatementtype == ESqlStatementType.sstselect)
                        {
                            TSourceToken prevst = ast.prevSolidToken();
                            if ((prevst.tokencode == TBaseType.rrw_as) || (prevst.tokencode == '(')|| (prevst.tokencode == ')'))
                            {
                                findNewStmt = false;
                            }
                        }
                    }
                }
                if (findNewStmt)
                {
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    gcurrentsqlstatement = lcStmt;
                    gcurrentsqlstatement.addtokentolist(ast);
                    continue;
                }
                else
                    gcurrentsqlstatement.addtokentolist(ast);

                break;
            }//case stsql

            case ststoredprocedure:{

                if (procedure_status[nestedProcedures] != stored_procedure_status.bodyend){
                    gcurrentsqlstatement.addtokentolist(ast);
                }

                switch (procedure_status[nestedProcedures]){
                    case cursor_declare:
                        if (ast.tokencode == ';'){
                            nestedProcedures--;
                            if (nestedProcedures < 0) nestedProcedures = 0;
                        }
                        break;
                    case start:
                        if ((ast.tokencode == TBaseType.rrw_as)||(ast.tokencode == TBaseType.rrw_is)){
                            // s1
                            if (sptype[nestedProcedures] != stored_procedure_type.create_trigger){
                                if ((sptype[0] == stored_procedure_type.package_spec) && (nestedProcedures > 0)){
                                 //when it's a package specification, only top level accept as/is
                                }else{
                                    procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                                    if (ast.searchToken("language",1) != null){
                                        // if as language is used in create function, then switch state to stored_procedure_status.body directly.
//                                        CREATE OR REPLACE FUNCTION THING.addressparse(p_addressline1 VARCHAR2) RETURN VARCHAR2 AUTHID DEFINER
//                                        as Language JAVA NAME 'AddressParser.parse(java.lang.String) return java.lang.String';
//                                        /
                                        if (nestedProcedures == 0){
                                          //  procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                            gst = EFindSqlStateType.stsql;
                                        }else{
                                            procedure_status[nestedProcedures] = stored_procedure_status.body;
                                            nestedProcedures--;
                                            //if (nestedProcedures > 0){ nestedProcedures--;}
                                        }

                                    }
                                }
                            }
                        }else if(ast.tokencode == TBaseType.rrw_begin){
                            // s4
                            if (sptype[nestedProcedures] == stored_procedure_type.create_trigger) waitingEnds[nestedProcedures]++;

                            if (nestedProcedures > 0) {nestedProcedures--;}
                            procedure_status[nestedProcedures] = stored_procedure_status.body;
                        }else if(ast.tokencode == TBaseType.rrw_end){
                            //s10
                            if((nestedProcedures > 0)&&(waitingEnds[nestedProcedures - 1] == 1)
                                    &&((sptype[nestedProcedures - 1] == stored_procedure_type.package_body)
                                        ||(sptype[nestedProcedures - 1] == stored_procedure_type.package_spec))){
                                nestedProcedures--;
                                procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                            }
                        }else if((ast.tokencode == TBaseType.rrw_procedure)||(ast.tokencode == TBaseType.rrw_function)){
                            //s3
                            if((nestedProcedures > 0)&&(waitingEnds[nestedProcedures] == 0)
                                    &&(procedure_status[nestedProcedures - 1] == stored_procedure_status.is_as)){
                                nestedProcedures--;
                                nestedProcedures++;
                                waitingEnds[nestedProcedures] = 0;
                                procedure_status[nestedProcedures] = stored_procedure_status.start;
                            }
                        }else if(ast.tokencode == TBaseType.rrw_oracle_cursor){
                            //s3, cursor in package specification
                            // PROCEDURE SM_23567_secondary_match;
                            // CURSOR active_lessors_cursor IS
                            if((nestedProcedures > 0)&&(waitingEnds[nestedProcedures] == 0)
                                    &&(procedure_status[nestedProcedures - 1] == stored_procedure_status.is_as)){
                                nestedProcedures--;
                                nestedProcedures++;
                                waitingEnds[nestedProcedures] = 0;
                                procedure_status[nestedProcedures] = stored_procedure_status.cursor_declare;
                            }
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger) && (ast.tokencode == TBaseType.rrw_declare) )
                        {
                            procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger)&&(ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) )
                        {
                          // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                           ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                           gst = EFindSqlStateType.stnormal;
                           doongetrawsqlstatementevent(gcurrentsqlstatement);

                           //make / a sqlplus cmd
                           gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                           gcurrentsqlstatement.addtokentolist(ast);
                           doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.create_trigger) )
                        {
                            if (ast.tokencode == TBaseType.rrw_trigger){
                                TSourceToken compoundSt =  ast.searchToken(TBaseType.rrw_oracle_compound, -1);
                                if (compoundSt != null){
                                    //it's trigger with compound trigger block
                                    procedure_status[nestedProcedures] = stored_procedure_status.body;
                                    waitingEnds[nestedProcedures]++;
                                }
                            }
                        }else if ((sptype[nestedProcedures] == stored_procedure_type.function)&&(ast.tokencode == TBaseType.rrw_teradata_using) )
                        {
                            if ((ast.searchToken("aggregate",-1) != null)||(ast.searchToken("pipelined",-1) != null)){
                                if (nestedProcedures == 0){
                                    //  procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                    gst = EFindSqlStateType.stsql;
                                }else{
                                    procedure_status[nestedProcedures] = stored_procedure_status.body;
                                    nestedProcedures--;
                                }
                            }

                        }else{
                            //other tokens, do nothing
                        }
                        break;
                    case is_as:
                        if((ast.tokencode == TBaseType.rrw_procedure)||(ast.tokencode == TBaseType.rrw_function)){
                          // s2
                          nestedProcedures++;
                          waitingEnds[nestedProcedures] = 0;
                          procedure_status[nestedProcedures] = stored_procedure_status.start;

                          if (nestedProcedures > stored_procedure_nested_level - 1){
                              gst = EFindSqlStateType.sterror;
                              nestedProcedures--;
                          }

                        }else if(ast.tokencode == TBaseType.rrw_begin){
                            // s5
                            if ((nestedProcedures == 0)&&
                                    ((sptype[nestedProcedures]==stored_procedure_type.package_body)
                                    ||(sptype[nestedProcedures]==stored_procedure_type.package_spec))){
                                //top level package or package body's BEGIN keyword already count,
                                // so don't increase waitingEnds[nestedProcedures] here

                            }else {
                                waitingEnds[nestedProcedures]++;
                            }
                            procedure_status[nestedProcedures] = stored_procedure_status.body;
                        }else if(ast.tokencode == TBaseType.rrw_end){
                            // s6
                            if ((nestedProcedures == 0)&&(waitingEnds[nestedProcedures] == 1)&&
                                    ((sptype[nestedProcedures]==stored_procedure_type.package_body)||(sptype[nestedProcedures]==stored_procedure_type.package_spec))){
                              procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                              waitingEnds[nestedProcedures]--;
                            }else{
                                waitingEnds[nestedProcedures]--;
                            }
                        }else if(ast.tokencode == TBaseType.rrw_case)
                        {
//                            if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                //this is not case after END
//                             waitingEnds[nestedProcedures]++;
//                            }
                            if (ast.searchToken(';',1) == null){
                                //this is not case before ;
                             waitingEnds[nestedProcedures]++;
                            }
                        }
                        else {
                            //other tokens, do nothing
                        }
                        break;
                    case body:
                        if ((ast.tokencode == TBaseType.rrw_begin))
                        {
                            waitingEnds[nestedProcedures]++;
                        } else if (ast.tokencode == TBaseType.rrw_if)
                        {
//                             if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                 //this is not if after END
//                              waitingEnds[nestedProcedures]++;
//                            }
                            if (ast.searchToken(';',2) == null){
                                //this is not if before ;

                                // 2015-02-27, change 1 to 2 make it able to detect label name after case
                                // like this: END CASE l1;
                             waitingEnds[nestedProcedures]++;
                            }
                        } else if(ast.tokencode == TBaseType.rrw_case)
                        {
//                            if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                //this is not case after END
//                             waitingEnds[nestedProcedures]++;
//                            }
                            if (ast.searchToken(';',2) == null){
                                //this is not case before ;
                                if (ast.searchToken(TBaseType.rrw_end,-1) == null){
                                    waitingEnds[nestedProcedures]++;
                                }
                            }
                        } else if(ast.tokencode == TBaseType.rrw_loop)
                        {
                            if (!((ast.searchToken(TBaseType.rrw_end,-1) != null)
                                &&(ast.searchToken(';',2) != null))){
                                // exclude loop like this:
                                // end loop [labelname];
                             waitingEnds[nestedProcedures]++;
                            }

//                            if (ast.searchToken(TBaseType.rrw_end,-1) == null){
//                                //this is not loop after END
//                             waitingEnds[nestedProcedures]++;
////                            }
////                            if (ast.searchToken(';',2) == null){
////                                //this is no loop before ;
////                             waitingEnds[nestedProcedures]++;
//                            } else if (ast.searchToken(TBaseType.rrw_null,1) != null){
//                                // mantis bug tracking system:   #65
//                                waitingEnds[nestedProcedures]++;
//                            }
                        }else if (ast.tokencode == TBaseType.rrw_end){
                            //foundEnd = true;
                            waitingEnds[nestedProcedures]--;
                            //if (waitingEnd < 0) { waitingEnd = 0;}
                            if (waitingEnds[nestedProcedures] == 0){
                                if (nestedProcedures == 0){
                                    // s7
                                    procedure_status[nestedProcedures] = stored_procedure_status.bodyend;
                                }else {
                                    // s71
                                    nestedProcedures--;
                                    procedure_status[nestedProcedures] = stored_procedure_status.is_as;
                                }
                            }
                        }else  if ((waitingEnds[nestedProcedures] == 0)&&(ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                        {
                          //sql ref: c:\prg\gsqlparser\Test\TestCases\oracle\createtrigger.sql, line 53
                           ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                           gst = EFindSqlStateType.stnormal;
                           doongetrawsqlstatementevent(gcurrentsqlstatement);

                           //make / a sqlplus cmd
                           gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                           gcurrentsqlstatement.addtokentolist(ast);
                           doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                        break;
                    case bodyend:
                        if ((ast.tokentype == ETokenType.ttslash)  && (ast.tokencode == TBaseType.sqlpluscmd) ) //and (prevst.NewlineIsLastTokenInTailerToken)) then
                        {
                          // TPlsqlStatementParse(asqlstatement).TerminatorToken := ast;
                           ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                           gst = EFindSqlStateType.stnormal;
                           doongetrawsqlstatementevent(gcurrentsqlstatement);

                           //make / a sqlplus cmd
                           gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                           gcurrentsqlstatement.addtokentolist(ast);
                           doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.tokentype == ETokenType.ttperiod)  && (sourcetokenlist.returnaftercurtoken(false)) && (sourcetokenlist.returnbeforecurtoken(false)))
                        {    // single dot at a seperate line
                           ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                           gst = EFindSqlStateType.stnormal;
                           doongetrawsqlstatementevent(gcurrentsqlstatement);

                           //make ttperiod a sqlplus cmd
                           gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                           gcurrentsqlstatement.addtokentolist(ast);
                           doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_package,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_procedure,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_function,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_create,1) != null)
                                &&((ast.searchToken(TBaseType.rrw_package,4) != null)||(ast.searchToken(TBaseType.rrw_package,5) != null))
                                &&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_create,1) != null)
                                &&((ast.searchToken(TBaseType.rrw_procedure,4) != null)
                                    ||(ast.searchToken(TBaseType.rrw_function,4) != null)
                                    ||(ast.searchToken(TBaseType.rrw_view,4) != null)
                                    ||(ast.searchToken(TBaseType.rrw_oracle_synonym,4) != null)
                                ||(ast.searchToken(TBaseType.rrw_trigger,4) != null)
                                    )
                                &&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_create,1) != null)&&(ast.searchToken(TBaseType.rrw_library,4) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_alter,1) != null)&&(ast.searchToken(TBaseType.rrw_trigger,2) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_select,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_call,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_commit,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_declare,1) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_grant,1) != null)&&
                                    (ast.searchToken(TBaseType.rrw_execute,2) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        } else  if ((ast.searchToken(TBaseType.rrw_alter,1) != null)&&
                                (ast.searchToken(TBaseType.rrw_table,2) != null)&&(!endBySlashOnly))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }else {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                        break;
                    case end:
                        break;
                    default:
                        break;
                }


                 if (ast.tokencode == TBaseType.sqlpluscmd)
                 {
                     //change tokencode back to keyword or TBaseType.ident, because sqlplus cmd
                     //in a sql statement(almost is plsql block) is not really a sqlplus cmd
                     int m = flexer.getkeywordvalue(ast.astext);
                     if (m != 0)
                     {ast.tokencode = m;}
                     else if (ast.tokentype == ETokenType.ttslash){
                         ast.tokencode = '/';
                     }
                     else
                     {ast.tokencode = TBaseType.ident;}
                 }

                final int wrapped_keyword_max_pos = 20;
                if ((ast.tokencode == TBaseType.rrw_wrapped)&&(ast.posinlist - gcurrentsqlstatement.sourcetokenlist.get(0).posinlist  < wrapped_keyword_max_pos)){
                    if (gcurrentsqlstatement instanceof TCommonStoredProcedureSqlStatement){
                       ((TCommonStoredProcedureSqlStatement)gcurrentsqlstatement).setWrapped(true);
                    }

                    if (gcurrentsqlstatement instanceof TPlsqlCreatePackage){
                        if (ast.prevSolidToken() != null){
                            ((TPlsqlCreatePackage) gcurrentsqlstatement).setPackageName(fparser.getNf().createObjectNameWithPart(ast.prevSolidToken()));
                        }
                    }
                }

                break;
            } //ststoredprocedure

        } //switch
      }//for

    //last statement
    if ((gcurrentsqlstatement != null) &&
        ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                    (gst == EFindSqlStateType.sterror)))
    {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
    }

  return syntaxErrors.size();
}

int domdxgetrawsqlstatements(){
    int waitingEnd = 0;
    boolean foundEnd = false;

    if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
    if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    TSourceToken lcprevsolidtoken = null,ast = null;

    for (int i=0 ; i < sourcetokenlist.size();i++)
      {

          if ( (ast != null ) && (ast.issolidtoken() ))
            lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

        switch(gst){
            case sterror:{
                if (ast.tokentype ==  ETokenType.ttsemicolon)
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    gst = EFindSqlStateType.stnormal;
                }
                else
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                break;
            } //sterror

            case stnormal:{
                if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                   || (ast.tokencode  == TBaseType.cmtslashstar)
                   || (ast.tokencode  == TBaseType.lexspace)
                   || (ast.tokencode  == TBaseType.lexnewline)
                   || (ast.tokentype  == ETokenType.ttsemicolon) )
                {
                   if (gcurrentsqlstatement != null)
                   {
                       gcurrentsqlstatement.addtokentolist(ast);
                   }

                   if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                   {
                       if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                       {
                          // ;;;; continuous semicolon,treat it as comment
                           ast.tokentype = ETokenType.ttsimplecomment;
                           ast.tokencode =  TBaseType.cmtdoublehyphen;
                       }
                   }

                   continue;
                }


                // find a tokentext to start sql or plsql mode
                gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                if (gcurrentsqlstatement != null)
                {
                    if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmdxscope)
                    {
                          gst = EFindSqlStateType.ststoredprocedure;
                          gcurrentsqlstatement.addtokentolist(ast);
                          foundEnd = false;
                          waitingEnd = 1;
                    }
                    else
                    {
                        gst = EFindSqlStateType.stsql;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                }else{
                    //error tokentext found

                    this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                            ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                    ast.tokentype = ETokenType.tttokenlizererrortoken;
                    gst = EFindSqlStateType.sterror;

                    gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                    gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                    gcurrentsqlstatement.addtokentolist(ast);

                }

                break;
            } // stnormal

            case stsql:{
                if (ast.tokentype == ETokenType.ttsemicolon)
                {
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast);
                    gcurrentsqlstatement.semicolonended = ast;
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    continue;
                }

                gcurrentsqlstatement.addtokentolist(ast);
                break;
            }//case stsql

            case ststoredprocedure:{
               if (ast.tokencode == TBaseType.rrw_if)
                {
                   waitingEnd++;
                }else if (ast.tokencode == TBaseType.rrw_case)
                {
                   waitingEnd++;
                }else if (ast.tokencode == TBaseType.rrw_scope)
                {
                    // ignore scrope after end keyword like this:
                    // end scope
                    if (lcprevsolidtoken.tokencode != TBaseType.rrw_end){
                        waitingEnd++;
                    }
                }else if (ast.tokencode == TBaseType.rrw_end){
                    foundEnd = true;
                    waitingEnd--;
                    if (waitingEnd < 0) { waitingEnd = 0;}
                }

                gcurrentsqlstatement.addtokentolist(ast);
                if ((ast.tokentype == ETokenType.ttsemicolon) && (waitingEnd == 0) && (foundEnd)){
                    gst = EFindSqlStateType.stnormal;
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                }

                break;
            } //ststoredprocedure
        } //switch
      }//for

    //last statement
    if ((gcurrentsqlstatement != null) &&
        ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) ||
                    (gst == EFindSqlStateType.sterror)))
    {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
    }

  return syntaxErrors.size();
}

int doimpalagetrawsqlstatements(){
   return dohivegetrawsqlstatements();
}

int dohivegetrawsqlstatements(){

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        TSourceToken lcprevsolidtoken = null,ast = null;

        for (int i=0 ; i < sourcetokenlist.size();i++)
          {

              if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

                ast = sourcetokenlist.get(i);
                sourcetokenlist.curpos = i;

              if (ast.tokencode == TBaseType.hive_CharSetName){
                  TSourceToken st1 = ast.searchToken(TBaseType.hive_CharSetLiteral,1);
                  if (st1 == null){
                      ast.tokencode = TBaseType.ident;
                  }
              }else  if (ast.tokencode == TBaseType.rrw_date) {
                  TSourceToken st1 = ast.nextSolidToken(); //ast.searchToken('(',1);
                  if (st1 != null) {
                      if (st1.tokencode == '(') {
                          ast.tokencode = TBaseType.rrw_hive_DATE_FUNCTION;
                      }
                    }
              }else  if (ast.tokencode == TBaseType.rrw_sort) {
                  TSourceToken st1 = ast.searchToken(TBaseType.rrw_by,1);
                  if (st1 == null) {
                      ast.tokencode = TBaseType.ident;
                  }
              }

            switch(gst){
                case sterror:{
                    if (ast.tokentype ==  ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                } //sterror

                case stnormal:{
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                       || (ast.tokencode  == TBaseType.cmtslashstar)
                       || (ast.tokencode  == TBaseType.lexspace)
                       || (ast.tokencode  == TBaseType.lexnewline)
                       || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                       if (gcurrentsqlstatement != null)
                       {
                           gcurrentsqlstatement.addtokentolist(ast);
                       }

                       if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                       {
                           if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                           {
                              // ;;;; continuous semicolon,treat it as comment
                               ast.tokentype = ETokenType.ttsimplecomment;
                               ast.tokencode =  TBaseType.cmtdoublehyphen;
                           }
                       }

                       continue;
                    }



                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (gcurrentsqlstatement != null)
                    {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                    }else{
                        //error tokentext found

                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }

                    break;
                } // stnormal

                case stsql:{
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }

                    gcurrentsqlstatement.addtokentolist(ast);
                    break;
                }//case stsql

            } //switch
          }//for

        //last statement
        if ((gcurrentsqlstatement != null) &&
            ((gst == EFindSqlStateType.stsql) ||(gst == EFindSqlStateType.sterror)))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

      return syntaxErrors.size();
    }

 int dosybasegetrawsqlstatements(){
    int errorcount = 0;
    int case_end_nest = 0;

    if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
    if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    int lcblocklevel = 0;
    int lctrycatchlevel = 0;
    TSourceToken lcprevsolidtoken = null,lcnextsolidtoken,lcnnextsolidtoken;
    TSourceToken ast = null;
    int i;
    boolean lcisendconversation, lcstillinsql;

    for (i=0 ; i < sourcetokenlist.size();i++)
      {

          if ( (ast != null ) && (ast.issolidtoken() ))
            lcprevsolidtoken = ast;

        ast = sourcetokenlist.get(i);
        sourcetokenlist.curpos = i;
        if ( ast.tokenstatus == ETokenStatus.tsignoredbygetrawstatement )
          {
            //tsignoredbygetrawstatement is set when cte is found in dbcmds.findcte function
            gcurrentsqlstatement.sourcetokenlist.add(ast);
            continue;
          }

        if ( gst == EFindSqlStateType.ststoredprocedurebody )
          {
            if ( !(
                (ast.tokencode == TBaseType.rrw_go)
                || (ast.tokencode == TBaseType.rrw_create)
                || (ast.tokencode == TBaseType.rrw_alter) )
              )
              {
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
              }
          }

        TCustomSqlStatement lcnextsqlstatement =  sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

        switch(gst){
          case sterror:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gcurrentsqlstatement = lcnextsqlstatement;
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  gst = EFindSqlStateType.stsql;
                }
              else if ( (ast.tokentype == ETokenType.ttsemicolon) )
                {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
                }
              else
                {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                break;
            }
          case stnormal :
            {
              if ( (ast.tokencode  == TBaseType.cmtdoublehyphen)
                 || (ast.tokencode  == TBaseType.cmtslashstar)
                 || (ast.tokencode  == TBaseType.lexspace)
                 || (ast.tokencode  == TBaseType.lexnewline)
                 || (ast.tokentype  == ETokenType.ttsemicolon) )
                {
                 if ( TBaseType.assigned(gcurrentsqlstatement) )
                   {
                     gcurrentsqlstatement.addtokentolist(ast);
                   }

                 if ( TBaseType.assigned(lcprevsolidtoken) && (ast.tokentype == ETokenType.ttsemicolon) )
                   {
                     if ( lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                       {
                        // ;;;; continuous semicolon,treat it as comment
                         ast.tokentype = ETokenType.ttsimplecomment;
                         ast.tokencode =  TBaseType.cmtdoublehyphen;
                       }
                     else
                       {
                       }

                   }

                 continue;
                }

              gcurrentsqlstatement = lcnextsqlstatement; //isstoredprocedure(ast,dbvendor,gst,sqlstatements);

              if ( TBaseType.assigned(gcurrentsqlstatement) )
                {
                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstmssqlcreateprocedure:
                    case sstmssqlcreatefunction:
                    case sstcreatetrigger:
                    case sstmssqlalterprocedure:
                    case   sstmssqlalterfunction:
                    case sstmssqlaltertrigger:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                      }
                    case sstmssqlbegintry:
                    case sstmssqlbegincatch:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.sttrycatch;
                        lctrycatchlevel = 0;
                          break;
                      }
                    case sstmssqlgo:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                          break;
                      }
                    default:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqllabel){
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }else{
                            gst = EFindSqlStateType.stsql;
                        }
                          break;
                      }
                  }    // case
                }
              else
                {
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                      gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                      gcurrentsqlstatement.addtokentolist(ast);
                      gst = EFindSqlStateType.stblock;
                    }
                  else
                    {
                      if ( sqlstatements.size() == 0 )
                        {
                          //first statement of mssql batch, treat it as exec sp
                          gst = EFindSqlStateType.stsql;
                          gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                            ast.tokencode = TBaseType.rrw_sybase_exce_proc_name;
                          //tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
// todo need previous line need to be implemented
                          gcurrentsqlstatement.addtokentolist(ast);
                        }
                      else if ( sqlstatements.get(sqlstatements.size() -1).sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                          // prev sql is go, treat it as exec sp
                          gst = EFindSqlStateType.stsql;
                          gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                            ast.tokencode = TBaseType.rrw_sybase_exce_proc_name;
                          //todo need to be implemented: tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
                          gcurrentsqlstatement.addtokentolist(ast);
                        }
                      else
                        {
                        }
                    }
                }


              if (  !TBaseType.assigned(gcurrentsqlstatement)  ) //error tokentext found
                {

//                  if ( TBaseType.assigned(ontokenlizertokenerror) )
//                    ontokenlizertokenerror(self,ast);
//                  incerrorcount;
//                  fsyntaxerrors[ferrorcount].hint = 'error when tokenlize';
//                  fsyntaxerrors[ferrorcount].errortype = spwarning;
//                  fsyntaxerrors[ferrorcount].tokentext = ast.sourcecode;
//                  fsyntaxerrors[ferrorcount].lines = ast.lines;
//                  fsyntaxerrors[ferrorcount].columns = ast.columns;// - length(fsyntaxerrors[ferrorcount].tokentext);

//                  errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                  errorcount = 1;
                    this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                            ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                  ast.tokentype = ETokenType.tttokenlizererrortoken;
                  gst = EFindSqlStateType.sterror;

                  gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                  gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                  gcurrentsqlstatement.addtokentolist(ast);
                }
                break;
            }
          case stblock:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                  else
                    {
                      lcnextsqlstatement = null;
                    }
                }

              if ( gst == EFindSqlStateType.stblock )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      // { [distributed] transaxtion/trans statement
                      // { dialog [ conversation ]
                      // { conversation timer
                      //  doesn't start block ({ .. })
                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( ! (TBaseType.mysametext(lcnextsolidtoken.astext,"tran")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"transaction")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"distributed")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"dialog")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"conversation")
                                  ) )
                           lcblocklevel++;
                        }
                      else
                        lcblocklevel++;

                    }
                  else if ( ast.tokencode == TBaseType.rrw_case )  // case ... }
                    lcblocklevel++;
                  else if ( ast.tokencode == TBaseType.rrw_end )
                    {

                      lcisendconversation = false;


                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( lcnextsolidtoken.tokencode == flexer.getkeywordvalue("conversation".toUpperCase()))
                            lcisendconversation = true; // } conversation statement
                        }


                      if ( ! lcisendconversation )
                        {

                          if ( lcblocklevel == 0 )
                            {
                              if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                                {
                                  if ( TBaseType.assigned(lcnextsolidtoken) )
                                    {
                                      if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                        {
                                          // { .. } else
                                          gst = EFindSqlStateType.stsql;
                                        }
                                      else if ( lcnextsolidtoken.tokentype == ETokenType.ttsemicolon )
                                        {
                                          lcnnextsolidtoken = sourcetokenlist.nextsolidtoken(lcnextsolidtoken.posinlist,1,false);
                                          if ( TBaseType.assigned(lcnnextsolidtoken) )
                                            {
                                              if ( lcnnextsolidtoken.tokencode == TBaseType.rrw_else )
                                                {
                                                  // { .. } else
                                                  gst = EFindSqlStateType.stsql;
                                                }
                                            }
                                        }
                                    }
                                }

                              if ( gst != EFindSqlStateType.stsql )
                                {
                                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                                  gst = EFindSqlStateType.stnormal;
                                }

                            }
                          else
                            {
                              lcblocklevel--;
                            }

                        }
                    }
                }
                break;
            }
          case sttrycatch:
            {

              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                  else
                    {
                      if (
                              (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry)
                              ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                        )
                         lctrycatchlevel++;
                      lcnextsqlstatement = null;
                    }
                }

              if ( gst == EFindSqlStateType.sttrycatch )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if ( (ast.tokencode == TBaseType.rrw_try) ||
                    (ast.tokencode == TBaseType.rrw_catch) )
                    {
                     // lcprevsolidtoken = sourcetokenlist.solidtokenbefore(i);
                      if ( TBaseType.assigned(lcprevsolidtoken) )
                        {
                          if ( lcprevsolidtoken.tokencode == TBaseType.rrw_end )
                            {
                              if ( lctrycatchlevel == 0 )
                                {
                                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                                  gst = EFindSqlStateType.stnormal;
                                }
                              else
                                lctrycatchlevel--;
                            }
                        }
                    }
                }
            break;
            }
          case stsql :
            {
              if ( (ast.tokentype == ETokenType.ttsemicolon)   )
                {
                  lcstillinsql = false;
                  if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                    {
                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                            {
                              // if ( expr stmt; else
                              gcurrentsqlstatement.addtokentolist(ast);
                              lcstillinsql = true;
                            }

                        }
                    }

                  if (  !lcstillinsql )
                    {
                      gst = EFindSqlStateType.stnormal;
                      gcurrentsqlstatement.addtokentolist(ast);
                      gcurrentsqlstatement.semicolonended = ast;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                }
              else if ( TBaseType.assigned(lcnextsqlstatement) )
                {

                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                        continue;
                    }

                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstmssqlif:
                    case sstmssqlwhile:
                      {
                        if ((lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                                ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stblock;
                                lcblocklevel = 1;
                                lcnextsqlstatement = null;
                                continue;

                        }
                        // if ( || while only contain one sql(not closed by {/} pair)
                        // so will still in if ( || while statement
                        else if ( gcurrentsqlstatement.dummytag == 1 )
                          {
                            // if ( cond ^stmt nextstmt (^ stands for current pos)
                            gcurrentsqlstatement.addtokentolist(ast);

                            if ( (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                              )
                                gcurrentsqlstatement.dummytag = 1;
                              else
                                gcurrentsqlstatement.dummytag = 0;


                            lcnextsqlstatement = null;
                            continue;
                          }
                      break;
                      }//
                    case sstmssqlalterqueue:
                      {
                       if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlexec )
                        {
                            // execute can't be used to delimite alter queue
                            gcurrentsqlstatement.addtokentolist(ast);


                            lcnextsqlstatement = null;
                            continue;

                        }
                      break;
                      }
                    case sstmssqlcreateschema:
                      {
                            gcurrentsqlstatement.addtokentolist(ast);
                            lcnextsqlstatement = null;
                            continue;
                      }
                  }//case

                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gcurrentsqlstatement = lcnextsqlstatement;
                  gcurrentsqlstatement.addtokentolist(ast);

                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstmssqlcreateprocedure:
                    case sstmssqlcreatefunction:
                    case sstcreatetrigger:
                    case sstmssqlalterprocedure:
                    case sstmssqlalterfunction:
                    case sstmssqlaltertrigger:
                      {
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                      }
                    case sstmssqlbegintry:
                    case sstmssqlbegincatch:
                      {
                        gst = EFindSqlStateType.sttrycatch;
                        lctrycatchlevel = 0;
                        break;
                      }
                    case sstmssqlgo:
                      {
                        gst = EFindSqlStateType.stnormal;
                        break;
                      }
                    default:
                      {
                        gst = EFindSqlStateType.stsql;
                        break;
                      }
                  }    // case

                }//TBaseType.assigned(lcnextsqlstatement)
              else if ( (ast.tokencode == TBaseType.rrw_begin) )
                {
                  if ( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                  || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                  )
                    {
                      // start block of if ( || while statement
                      gst = EFindSqlStateType.stblock;
                      lcblocklevel = 0;
                      gcurrentsqlstatement.addtokentolist(ast);
                    }
                   else if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqldeclare )
                      {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.stblock;
                      }
                  else
                    {
                      gcurrentsqlstatement.addtokentolist(ast);
                    }
                }
              else if ( (ast.tokencode == TBaseType.rrw_case) ){
                  case_end_nest++;
                  gcurrentsqlstatement.addtokentolist(ast);
              }
              else if ( (ast.tokencode == TBaseType.rrw_end) ){
                  if (case_end_nest > 0){
                      case_end_nest--;
                  }
                  gcurrentsqlstatement.addtokentolist(ast);
              }
              else if ( (ast.tokencode == TBaseType.rrw_else) )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  // if ( cond stmt ^else stmt
                  if (( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                  || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                  ) && (case_end_nest == 0))
                    gcurrentsqlstatement.dummytag = 1; // reduce to 1 while stmt after else is found: if ( cond stmt ^else stmt
                }
              else
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlset){
                        if (ast.tokencode == TBaseType.rrw_on){
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                  }
                }
             break;
            }
          case ststoredprocedure:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                  else
                    {
                      gst = EFindSqlStateType.ststoredprocedurebody;
                      gcurrentsqlstatement.addtokentolist(ast);

                      lcnextsqlstatement = null;
                    }
                }

              if ( gst == EFindSqlStateType.ststoredprocedure )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      gst = EFindSqlStateType.stblock;
                    }
                }
             break;
            } //stplsql
          case ststoredprocedurebody:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  switch(lcnextsqlstatement.sqlstatementtype){    //
                    case sstmssqlgo:
                      {
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gcurrentsqlstatement = lcnextsqlstatement;
                          gcurrentsqlstatement.addtokentolist(ast);
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gst = EFindSqlStateType.stnormal;
                          break;
                      }
                    case sstmssqlcreateprocedure:
                    case sstmssqlcreatefunction:
                    case sstcreatetrigger:
                    case sstmssqlalterprocedure:
                    case sstmssqlalterfunction:
                    case sstmssqlaltertrigger:
                      {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedure;
                          break;
                      }
                    default:
                      {
                        lcnextsqlstatement = null;
                          break;
                      }
                  }//case
                }

              if ( gst == EFindSqlStateType.ststoredprocedurebody )
                gcurrentsqlstatement.addtokentolist(ast);
            }
          break;
        } //case
      } //for


    //last statement
    if ( TBaseType.assigned(gcurrentsqlstatement) &&  (gst != EFindSqlStateType.stnormal))
      {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
      }



    return errorcount;

}

int doinformixgetrawsqlstatements(){
    int errorcount = 0;
    int case_end_nest = 0;

    if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
    if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    int lcblocklevel = 0;
    int lctrycatchlevel = 0;
    TSourceToken lcprevsolidtoken = null,lcnextsolidtoken,lcnnextsolidtoken;
    TSourceToken ast = null;
    int i;
    boolean lcisendconversation, lcstillinsql;

    for (i=0 ; i < sourcetokenlist.size();i++)
      {

          if ( (ast != null ) && (ast.issolidtoken() ))
            lcprevsolidtoken = ast;

        ast = sourcetokenlist.get(i);
        sourcetokenlist.curpos = i;
        if ( ast.tokenstatus == ETokenStatus.tsignoredbygetrawstatement )
          {
            //tsignoredbygetrawstatement is set when cte is found in dbcmds.findcte function
            gcurrentsqlstatement.sourcetokenlist.add(ast);
            continue;
          }

        if ( gst == EFindSqlStateType.ststoredprocedurebody )
          {
            if ( !(
                (ast.tokencode == TBaseType.rrw_go)
                || (ast.tokencode == TBaseType.rrw_create)
                || (ast.tokencode == TBaseType.rrw_alter) )
              )
              {
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
              }
          }

        TCustomSqlStatement lcnextsqlstatement =  sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

        switch(gst){
          case sterror:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gcurrentsqlstatement = lcnextsqlstatement;
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  gst = EFindSqlStateType.stsql;
                }
              else if ( (ast.tokentype == ETokenType.ttsemicolon) )
                {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
                }
              else
                {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                break;
            }
          case stnormal :
            {
              if ( (ast.tokencode  == TBaseType.cmtdoublehyphen)
                 || (ast.tokencode  == TBaseType.cmtslashstar)
                 || (ast.tokencode  == TBaseType.lexspace)
                 || (ast.tokencode  == TBaseType.lexnewline)
                 || (ast.tokentype  == ETokenType.ttsemicolon) )
                {
                 if ( TBaseType.assigned(gcurrentsqlstatement) )
                   {
                     gcurrentsqlstatement.addtokentolist(ast);
                   }

                 if ( TBaseType.assigned(lcprevsolidtoken) && (ast.tokentype == ETokenType.ttsemicolon) )
                   {
                     if ( lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                       {
                        // ;;;; continuous semicolon,treat it as comment
                         ast.tokentype = ETokenType.ttsimplecomment;
                         ast.tokencode =  TBaseType.cmtdoublehyphen;
                       }
                     else
                       {
                       }

                   }

                 continue;
                }

              gcurrentsqlstatement = lcnextsqlstatement; //isstoredprocedure(ast,dbvendor,gst,sqlstatements);

              if ( TBaseType.assigned(gcurrentsqlstatement) )
                {
                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstinformixCreateProcedure:
                    case sstinformixCreateFunction:
                    case sstcreatetrigger:
                    case sstinformixAlterProcedure:
                    case sstinformixAlterFunction:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                      }
                    case sstmssqlbegintry:
                    case sstmssqlbegincatch:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.sttrycatch;
                        lctrycatchlevel = 0;
                          break;
                      }
                    case sstmssqlgo:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                          break;
                      }
                    case sstinformixExecute:
                    {
                        gst = EFindSqlStateType.stExec;
                        break;
                    }
                    default:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.stsql;
                          break;
                      }
                  }    // case
                }
              else
                {
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                      gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                      gcurrentsqlstatement.addtokentolist(ast);
                      gst = EFindSqlStateType.stblock;
                    }
                  else
                    {
                      if ( sqlstatements.size() == 0 )
                        {
                          //first statement of mssql batch, treat it as exec sp
                          gst = EFindSqlStateType.stsql;
                          gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                          //tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
// todo need previous line need to be implemented
                          gcurrentsqlstatement.addtokentolist(ast);
                        }
                      else if ( sqlstatements.get(sqlstatements.size() -1).sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                          // prev sql is go, treat it as exec sp
                          gst = EFindSqlStateType.stsql;
                          gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                          //todo need to be implemented: tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
                          gcurrentsqlstatement.addtokentolist(ast);
                        }
                      else
                        {
                        }
                    }
                }


              if (  !TBaseType.assigned(gcurrentsqlstatement)  ) //error tokentext found
                {

//                  if ( TBaseType.assigned(ontokenlizertokenerror) )
//                    ontokenlizertokenerror(self,ast);
//                  incerrorcount;
//                  fsyntaxerrors[ferrorcount].hint = 'error when tokenlize';
//                  fsyntaxerrors[ferrorcount].errortype = spwarning;
//                  fsyntaxerrors[ferrorcount].tokentext = ast.sourcecode;
//                  fsyntaxerrors[ferrorcount].lines = ast.lines;
//                  fsyntaxerrors[ferrorcount].columns = ast.columns;// - length(fsyntaxerrors[ferrorcount].tokentext);

//                  errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                  errorcount = 1;
                    this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                            ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                  ast.tokentype = ETokenType.tttokenlizererrortoken;
                  gst = EFindSqlStateType.sterror;

                  gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                  gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                  gcurrentsqlstatement.addtokentolist(ast);
                }
                break;
            }
          case stExec:
          {
              gcurrentsqlstatement.sourcetokenlist.add(ast);
              if  (ast.tokentype == ETokenType.ttsemicolon){
                  gst = EFindSqlStateType.stnormal;
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
              }
              break;
          }
          case stblock:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                  else
                    {
                      lcnextsqlstatement = null;
                    }
                }

              if ( gst == EFindSqlStateType.stblock )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      // { [distributed] transaxtion/trans statement
                      // { dialog [ conversation ]
                      // { conversation timer
                      //  doesn't start block ({ .. })
                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( ! (TBaseType.mysametext(lcnextsolidtoken.astext,"tran")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"transaction")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"distributed")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"dialog")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"conversation")
                                  ) )
                           lcblocklevel++;
                        }
                      else
                        lcblocklevel++;

                    }
                  else if ( ast.tokencode == TBaseType.rrw_case )  // case ... }
                    lcblocklevel++;
                  else if ( ast.tokencode == TBaseType.rrw_end )
                    {

                      lcisendconversation = false;


                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( lcnextsolidtoken.tokencode == flexer.getkeywordvalue("conversation".toUpperCase()))
                            lcisendconversation = true; // } conversation statement
                        }


                      if ( ! lcisendconversation )
                        {

                          if ( lcblocklevel == 0 )
                            {
                              if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                                {
                                  if ( TBaseType.assigned(lcnextsolidtoken) )
                                    {
                                      if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                        {
                                          // { .. } else
                                          gst = EFindSqlStateType.stsql;
                                        }
                                      else if ( lcnextsolidtoken.tokentype == ETokenType.ttsemicolon )
                                        {
                                          lcnnextsolidtoken = sourcetokenlist.nextsolidtoken(lcnextsolidtoken.posinlist,1,false);
                                          if ( TBaseType.assigned(lcnnextsolidtoken) )
                                            {
                                              if ( lcnnextsolidtoken.tokencode == TBaseType.rrw_else )
                                                {
                                                  // { .. } else
                                                  gst = EFindSqlStateType.stsql;
                                                }
                                            }
                                        }
                                    }
                                }

                              if ( gst != EFindSqlStateType.stsql )
                                {
                                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                                  gst = EFindSqlStateType.stnormal;
                                }

                            }
                          else
                            {
                              lcblocklevel--;
                            }

                        }
                    }
                }
                break;
            }
          case sttrycatch:
            {

              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                  else
                    {
                      if (
                              (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry)
                              ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                        )
                         lctrycatchlevel++;
                      lcnextsqlstatement = null;
                    }
                }

              if ( gst == EFindSqlStateType.sttrycatch )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if ( (ast.tokencode == TBaseType.rrw_try) ||
                    (ast.tokencode == TBaseType.rrw_catch) )
                    {
                     // lcprevsolidtoken = sourcetokenlist.solidtokenbefore(i);
                      if ( TBaseType.assigned(lcprevsolidtoken) )
                        {
                          if ( lcprevsolidtoken.tokencode == TBaseType.rrw_end )
                            {
                              if ( lctrycatchlevel == 0 )
                                {
                                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                                  gst = EFindSqlStateType.stnormal;
                                }
                              else
                                lctrycatchlevel--;
                            }
                        }
                    }
                }
            break;
            }
          case stsql :
            {
              if ( (ast.tokentype == ETokenType.ttsemicolon)   )
                {
                  lcstillinsql = false;
                  if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                    {
                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                            {
                              // if ( expr stmt; else
                              gcurrentsqlstatement.addtokentolist(ast);
                              lcstillinsql = true;
                            }

                        }
                    }

                  if (  !lcstillinsql )
                    {
                      gst = EFindSqlStateType.stnormal;
                      gcurrentsqlstatement.addtokentolist(ast);
                      gcurrentsqlstatement.semicolonended = ast;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                }
              else if ( TBaseType.assigned(lcnextsqlstatement) )
                {

                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                        continue;
                    }

                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstmssqlif:
                    case sstmssqlwhile:
                      {
                        if ((lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                                ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry))
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stblock;
                                lcblocklevel = 1;
                                lcnextsqlstatement = null;
                                continue;

                        }
                        // if ( || while only contain one sql(not closed by {/} pair)
                        // so will still in if ( || while statement
                        else if ( gcurrentsqlstatement.dummytag == 1 )
                          {
                            // if ( cond ^stmt nextstmt (^ stands for current pos)
                            gcurrentsqlstatement.addtokentolist(ast);

                            if ( (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                              )
                                gcurrentsqlstatement.dummytag = 1;
                              else
                                gcurrentsqlstatement.dummytag = 0;


                            lcnextsqlstatement = null;
                            continue;
                          }
                      break;
                      }//
                    case sstmssqlalterqueue:
                      {
                       if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlexec )
                        {
                            // execute can't be used to delimite alter queue
                            gcurrentsqlstatement.addtokentolist(ast);


                            lcnextsqlstatement = null;
                            continue;

                        }
                      break;
                      }
                    case sstmssqlcreateschema:
                      {
                            gcurrentsqlstatement.addtokentolist(ast);
                            lcnextsqlstatement = null;
                            continue;
                      }
                  }//case

                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gcurrentsqlstatement = lcnextsqlstatement;
                  gcurrentsqlstatement.addtokentolist(ast);

                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstinformixCreateProcedure:
                    case sstinformixCreateFunction:
                    case sstcreatetrigger:
                    case sstinformixAlterProcedure:
                    case sstinformixAlterFunction:
                      {
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                      }
                    case sstmssqlbegintry:
                    case sstmssqlbegincatch:
                      {
                        gst = EFindSqlStateType.sttrycatch;
                        lctrycatchlevel = 0;
                        break;
                      }
                    case sstmssqlgo:
                      {
                        gst = EFindSqlStateType.stnormal;
                        break;
                      }
                    default:
                      {
                        gst = EFindSqlStateType.stsql;
                        break;
                      }
                  }    // case

                }//TBaseType.assigned(lcnextsqlstatement)
              else if ( (ast.tokencode == TBaseType.rrw_begin) )
                {
                  if ( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                  || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                  )
                    {
                      // start block of if ( || while statement
                      gst = EFindSqlStateType.stblock;
                      lcblocklevel = 0;
                      gcurrentsqlstatement.addtokentolist(ast);
                    }
                   else if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqldeclare )
                      {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.stblock;
                      }
                  else
                    {
                      gcurrentsqlstatement.addtokentolist(ast);
                    }
                }
              else if ( (ast.tokencode == TBaseType.rrw_case) ){
                  case_end_nest++;
                  gcurrentsqlstatement.addtokentolist(ast);
              }
              else if ( (ast.tokencode == TBaseType.rrw_end) ){
                  if (case_end_nest > 0){
                      case_end_nest--;
                  }
                  gcurrentsqlstatement.addtokentolist(ast);
              }
              else if ( (ast.tokencode == TBaseType.rrw_else) )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  // if ( cond stmt ^else stmt
                  if (( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                  || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                  ) && (case_end_nest == 0))
                    gcurrentsqlstatement.dummytag = 1; // reduce to 1 while stmt after else is found: if ( cond stmt ^else stmt
                }
              else
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                }
             break;
            }
          case ststoredprocedure:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                  else
                    {
                      gst = EFindSqlStateType.ststoredprocedurebody;
                      gcurrentsqlstatement.addtokentolist(ast);

                      lcnextsqlstatement = null;
                    }
                }

              if ( gst == EFindSqlStateType.ststoredprocedure )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if (( ast.tokencode == TBaseType.rrw_procedure )
                      ||( ast.tokencode == TBaseType.rrw_function ))
                    {
                        if (ast.searchToken(TBaseType.rrw_end,-1) != null)
                            gst = EFindSqlStateType.stsql;
                    }
                }
             break;
            } //stplsql
          case ststoredprocedurebody:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  switch(lcnextsqlstatement.sqlstatementtype){    //
                    case sstmssqlgo:
                      {
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gcurrentsqlstatement = lcnextsqlstatement;
                          gcurrentsqlstatement.addtokentolist(ast);
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gst = EFindSqlStateType.stnormal;
                          break;
                      }
                    case sstinformixCreateProcedure:
                    case sstinformixCreateFunction:
                    case sstcreatetrigger:
                    case sstinformixAlterProcedure:
                    case sstinformixAlterFunction:
                      {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedure;
                          break;
                      }
                    default:
                      {
                        lcnextsqlstatement = null;
                          break;
                      }
                  }//case
                }

              if ( gst == EFindSqlStateType.ststoredprocedurebody )
                gcurrentsqlstatement.addtokentolist(ast);
            }
          break;
        } //case
      } //for


    //last statement
    if ( TBaseType.assigned(gcurrentsqlstatement) &&  (gst != EFindSqlStateType.stnormal))
      {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
      }



    return errorcount;

}

    int dosoqlgetrawsqlstatements(){
        int errorcount = 0;
        int case_end_nest = 0;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        int lcblocklevel = 0;
        int lctrycatchlevel = 0;
        TSourceToken lcprevsolidtoken = null,lcnextsolidtoken,lcnnextsolidtoken;
        TSourceToken ast = null;
        int i,lcMergeInSelectNested = 0;
        boolean lcisendconversation, lcstillinsql,lcMergeInSelect = false;

        for (i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (lcMergeInSelect){
                if (ast.tokencode == '(') lcMergeInSelectNested++;
                if (ast.tokencode == ')'){
                    lcMergeInSelectNested--;
                    if (lcMergeInSelectNested == 0){
                        lcMergeInSelect = false;
                    }
                }
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }
            if ( ast.tokenstatus == ETokenStatus.tsignoredbygetrawstatement )
            {
                //tsignoredbygetrawstatement is set when cte is found in dbcmds.findcte function
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }

            if (ast.tokencode == TBaseType.rrw_update) {
                TSourceToken st1 = ast.nextSolidToken();
                if (st1 != null) {
                    if (st1.toString().equalsIgnoreCase("tracking")
                    ||st1.toString().equalsIgnoreCase("viewstat")){ //UPDATE TRACKING|VIEWSTAT
                        ast.tokencode  = TBaseType.rrw_soql_update_tracking;
                    }
                }
            }

            if (dbVendor == EDbVendor.dbvopenedge){
                if (ast.tokencode == TBaseType.rrw_order){
                    TSourceToken st1 = ast.searchToken(TBaseType.rrw_by,1);
                    if (st1 == null) {
                        ast.tokencode = TBaseType.ident;
                    }
                }else if (ast.tokencode == TBaseType.rrw_with){
                    TSourceToken st1 = ast.searchToken(TBaseType.rrw_check,1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_openedge_with_check;
                    }
                }
            }

            if ( gst == EFindSqlStateType.ststoredprocedurebody )
            {
                if ( !(
                        (ast.tokencode == TBaseType.rrw_go)
                                || (ast.tokencode == TBaseType.rrw_create)
                                || (ast.tokencode == TBaseType.rrw_alter) )
                )
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                    continue;
                }
            }

            TCustomSqlStatement lcnextsqlstatement =  sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

            switch(gst){
                case sterror:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        gst = EFindSqlStateType.stsql;
                    }
                    else if ( (ast.tokentype == ETokenType.ttsemicolon) )
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                }
                case stnormal :
                {
                    if ( (ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if ( TBaseType.assigned(gcurrentsqlstatement) )
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ( TBaseType.assigned(lcprevsolidtoken) && (ast.tokentype == ETokenType.ttsemicolon) )
                        {
                            if ( lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                            else
                            {
                            }

                        }

                        continue;
                    }

                    gcurrentsqlstatement = lcnextsqlstatement; //isstoredprocedure(ast,dbvendor,gst,sqlstatements);

                    if ( TBaseType.assigned(gcurrentsqlstatement) )
                    {
                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                                //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case   sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstmssqlbegintry:
                            case sstmssqlbegincatch:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.sttrycatch;
                                lctrycatchlevel = 0;
                                break;
                            }
                            case sstmssqlgo:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            default:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stsql;
                                break;
                            }
                        }    // case
                    }
                    else
                    {
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stblock;
                        }
                        else
                        {
                            if ( sqlstatements.size() == 0 )
                            {
                                //first statement of mssql batch, treat it as exec sp
                                gst = EFindSqlStateType.stsql;
                                gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                                //tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
// todo need previous line need to be implemented
                                gcurrentsqlstatement.addtokentolist(ast);
                            }
                            else if ( sqlstatements.get(sqlstatements.size() -1).sqlstatementtype == ESqlStatementType.sstmssqlgo )
                            {
                                // prev sql is go, treat it as exec sp
                                gst = EFindSqlStateType.stsql;
                                gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                                //todo need to be implemented: tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
                                gcurrentsqlstatement.addtokentolist(ast);
                            }
                            else
                            {
                            }
                        }
                    }


                    if (  !TBaseType.assigned(gcurrentsqlstatement)  ) //error tokentext found
                    {

//                        errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                        errorcount = 1;
                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    break;
                }
                case stblock:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else
                        {
                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.stblock )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            // { [distributed] transaxtion/trans statement
                            // { dialog [ conversation ]
                            // { conversation timer
                            //  doesn't start block ({ .. })
                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( ! (TBaseType.mysametext(lcnextsolidtoken.astext,"tran")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"transaction")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"distributed")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"dialog")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"conversation")
                                ) )
                                    lcblocklevel++;
                            }
                            else
                                lcblocklevel++;

                        }
                        else if ( ast.tokencode == TBaseType.rrw_case )  // case ... }
                            lcblocklevel++;
                        else if ( ast.tokencode == TBaseType.rrw_end )
                        {

                            lcisendconversation = false;


                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( lcnextsolidtoken.tokencode == flexer.getkeywordvalue("conversation".toUpperCase()))
                                    lcisendconversation = true; // } conversation statement
                            }


                            if ( ! lcisendconversation )
                            {

                                if ( lcblocklevel == 0 )
                                {
                                    if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                                    {
                                        if ( TBaseType.assigned(lcnextsolidtoken) )
                                        {
                                            if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                            {
                                                // { .. } else
                                                gst = EFindSqlStateType.stsql;
                                            }
                                            else if ( lcnextsolidtoken.tokentype == ETokenType.ttsemicolon )
                                            {
                                                lcnnextsolidtoken = sourcetokenlist.nextsolidtoken(lcnextsolidtoken.posinlist,1,false);
                                                if ( TBaseType.assigned(lcnnextsolidtoken) )
                                                {
                                                    if ( lcnnextsolidtoken.tokencode == TBaseType.rrw_else )
                                                    {
                                                        // { .. } else
                                                        gst = EFindSqlStateType.stsql;
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    if ( gst != EFindSqlStateType.stsql )
                                    {
                                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                                        gst = EFindSqlStateType.stnormal;
                                    }

                                }
                                else
                                {
                                    lcblocklevel--;
                                }

                            }
                        }
                    }
                    break;
                }
                case sttrycatch:
                {

                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else
                        {
                            if (
                                    (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry)
                                            ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                            )
                                lctrycatchlevel++;
                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.sttrycatch )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( (ast.tokencode == TBaseType.rrw_try) ||
                                (ast.tokencode == TBaseType.rrw_catch) )
                        {
                            // lcprevsolidtoken = sourcetokenlist.solidtokenbefore(i);
                            if ( TBaseType.assigned(lcprevsolidtoken) )
                            {
                                if ( lcprevsolidtoken.tokencode == TBaseType.rrw_end )
                                {
                                    if ( lctrycatchlevel == 0 )
                                    {
                                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                                        gst = EFindSqlStateType.stnormal;
                                    }
                                    else
                                        lctrycatchlevel--;
                                }
                            }
                        }
                    }
                    break;
                }
                case stsql :
                {
                    if ( (ast.tokentype == ETokenType.ttsemicolon)   )
                    {
                        lcstillinsql = false;
                        if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                        {
                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                {
                                    // if ( expr stmt; else
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    lcstillinsql = true;
                                }

                            }
                        }

                        if (  !lcstillinsql )
                        {
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gcurrentsqlstatement.semicolonended = ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }

                    }
                    else if ( TBaseType.assigned(lcnextsqlstatement) )
                    {

                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                            continue;
                        }

                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlif:
                            case sstmssqlwhile:
                            {
                                if ((lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                                        ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry))
                                {
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    gst = EFindSqlStateType.stblock;
                                    lcblocklevel = 1;
                                    lcnextsqlstatement = null;
                                    continue;

                                }
                                // if ( || while only contain one sql(not closed by {/} pair)
                                // so will still in if ( || while statement
                                else if ( gcurrentsqlstatement.dummytag == 1 )
                                {
                                    // if ( cond ^stmt nextstmt (^ stands for current pos)
                                    gcurrentsqlstatement.addtokentolist(ast);

                                    if ( (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                                    )
                                        gcurrentsqlstatement.dummytag = 1;
                                    else
                                        gcurrentsqlstatement.dummytag = 0;


                                    lcnextsqlstatement = null;
                                    continue;
                                }
                                break;
                            }//
                            case sstmssqlalterqueue:
                            {
                                if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlexec )
                                {
                                    // execute can't be used to delimite alter queue
                                    gcurrentsqlstatement.addtokentolist(ast);


                                    lcnextsqlstatement = null;
                                    continue;

                                }
                                break;
                            }
                            case sstmssqlcreateschema:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                lcnextsqlstatement = null;
                                continue;
                            }
                        }//case

                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);

                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                                //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstmssqlbegintry:
                            case sstmssqlbegincatch:
                            {
                                gst = EFindSqlStateType.sttrycatch;
                                lctrycatchlevel = 0;
                                break;
                            }
                            case sstmssqlgo:
                            {
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            default:
                            {
                                gst = EFindSqlStateType.stsql;
                                break;
                            }
                        }    // case

                    }//TBaseType.assigned(lcnextsqlstatement)
                    else if ( (ast.tokencode == TBaseType.rrw_begin) )
                    {
                        if ( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                        )
                        {
                            // start block of if ( || while statement
                            gst = EFindSqlStateType.stblock;
                            lcblocklevel = 0;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                        else if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqldeclare )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stblock;
                        }
                        else
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_case) ){
                        case_end_nest++;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_end) ){
                        if (case_end_nest > 0){
                            case_end_nest--;
                        }
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_else) )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        // if ( cond stmt ^else stmt
                        if (( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                        ) && (case_end_nest == 0))
                            gcurrentsqlstatement.dummytag = 1; // reduce to 1 while stmt after else is found: if ( cond stmt ^else stmt
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    break;
                }
                case ststoredprocedure:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else
                        {
                            gst = EFindSqlStateType.ststoredprocedurebody;
                            gcurrentsqlstatement.addtokentolist(ast);

                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.ststoredprocedure )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            gst = EFindSqlStateType.stblock;
                        }
                    }
                    break;
                } //stplsql
                case ststoredprocedurebody:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        switch(lcnextsqlstatement.sqlstatementtype){    //
                            case sstmssqlgo:
                            {
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                                //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstcreateview:
                            case sstcreatetable:
                            {

                                boolean readForNewStmt = false;
                                TSourceToken st1 = ast.searchToken(';',-1);
                                if (st1 != null) {
                                    TSourceToken st2 = ast.searchToken(TBaseType.rrw_end,-2);
                                    if (st2 != null){
                                        readForNewStmt = true;
                                    }
                                }

                                if (readForNewStmt){
                                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                                    gcurrentsqlstatement = lcnextsqlstatement;
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    gst = EFindSqlStateType.stsql;
                                }else{
                                    lcnextsqlstatement = null;
                                }
                                break;
                            }
                            default:
                            {
                                lcnextsqlstatement = null;
                                break;
                            }
                        }//case
                    }

                    if ( gst == EFindSqlStateType.ststoredprocedurebody )
                        gcurrentsqlstatement.addtokentolist(ast);
                }
                break;
            } //case
        } //for


        //last statement
        if ( TBaseType.assigned(gcurrentsqlstatement) &&  (gst != EFindSqlStateType.stnormal))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }



        return errorcount;

    }

    int domssqlgetrawsqlstatements(){
        int errorcount = 0;
        int case_end_nest = 0;

        if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
        if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        int lcblocklevel = 0;
        int lctrycatchlevel = 0;
        TSourceToken lcprevsolidtoken = null,lcnextsolidtoken,lcnnextsolidtoken;
        TSourceToken ast = null;
        int i,lcMergeInSelectNested = 0;
        boolean lcisendconversation, lcstillinsql,lcMergeInSelect = false;

        for (i=0 ; i < sourcetokenlist.size();i++)
        {

            if ( (ast != null ) && (ast.issolidtoken() ))
                lcprevsolidtoken = ast;

            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (ast.tokencode == TBaseType.rrw_for){
                TSourceToken st1 = ast.nextSolidToken();
                if ((st1 != null) && (st1.tokencode == TBaseType.rrw_system_time)){
                    ast.tokencode = TBaseType.rw_for_system_time;
                }
            }

            if (lcMergeInSelect){
                if (ast.tokencode == '(') lcMergeInSelectNested++;
                if (ast.tokencode == ')'){
                    lcMergeInSelectNested--;
                    if (lcMergeInSelectNested == 0){
                        lcMergeInSelect = false;
                    }
                }
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }
            if ( ast.tokenstatus == ETokenStatus.tsignoredbygetrawstatement )
            {
                //tsignoredbygetrawstatement is set when cte is found in dbcmds.findcte function
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
            }

            if (ast.tokencode == TBaseType.rrw_minus){
                TSourceToken st1 = ast.searchToken('(',1);
                if (st1 == null){
                    st1 = ast.searchToken(TBaseType.rrw_select,1);
                    if (st1 == null){
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_merge){
                TSourceToken st1 = ast.nextSolidToken();
                if (st1.tokencode == TBaseType.rrw_join){
                    ast.tokencode = TBaseType.rrw_merge2_sqlserver;
                }
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '(')){
                    lcMergeInSelect = true;
                    lcMergeInSelectNested++;
                    gcurrentsqlstatement.addtokentolist(ast);
                    continue;
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_value){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_value;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_modify){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_modify;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_query){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_query;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_exist){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_exist;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_nodes){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '.')){
                    TSourceToken st1 = ast.searchToken('(',1);
                    if (st1 != null) {
                        ast.tokencode = TBaseType.rrw_xml_nodes;
                    }
                }
            }else if (ast.tokencode == TBaseType.rrw_check){
                if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == TBaseType.rrw_with)){
                    lcprevsolidtoken.tokencode = TBaseType.rrw_sqlserver_check_with;
                }
//            }else if (ast.tokencode == TBaseType.rrw_for){
//                TSourceToken st1 = ast.nextSolidToken();
//                if ((st1 != null) && (st1.tokencode == TBaseType.rrw_system_time)){
//                    ast.tokencode = TBaseType.rw_for_system_time;
//                }
            }else if (ast.tokencode == TBaseType.rrw_sqlserver_next){
                TSourceToken st1 = ast.nextSolidToken();
                if ((st1 != null) && (st1.tokencode == '.')){
                    ast.tokencode = TBaseType.ident;
                }
            }else if (ast.tokencode == TBaseType.rrw_fetch){
                if ((lcprevsolidtoken != null)&&((lcprevsolidtoken.tokencode == TBaseType.rrw_sqlserver_row)||(lcprevsolidtoken.tokencode == TBaseType.rrw_sqlserver_rows))){
                      TSourceToken prev2 = lcprevsolidtoken.searchToken(TBaseType.rrw_open,-1);
                      if (prev2 == null){
                          ast.tokencode = TBaseType.rrw_sqlserver_offset_fetch;
                      }else{
                          // don't turn fetch to TBaseType.rrw_sqlserver_offset_fetch
                          //open row
                          //fetch row into @msg
                      }

                }
            }else if((ast.tokencode == TBaseType.rrw_exec)||(ast.tokencode == TBaseType.rrw_execute)){
                // search ;2 after execute
                // EXECUTE ApxSQL_Test_Triggers_Add;2  @TableName, @AddFlag
                int searchRange = 4;
                TSourceToken endTokenInSameLine = ast.searchTokenAtTheEndOfSameLine();
                if (endTokenInSameLine != null){
                    searchRange = endTokenInSameLine.posinlist - ast.posinlist;
                }
                TSourceToken st1 = ast.searchToken(';',searchRange);
                if (st1 != null){
                    TSourceToken numSt = st1.nextSolidToken();
                    if ((numSt != null)&&(numSt.tokentype == ETokenType.ttnumber)){
                        st1.tokencode = TBaseType.rrw_sqlserver_semicolon_module_number;
                        st1.tokentype = ETokenType.ttidentifier;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_sqlserver_trim){
                TSourceToken st1 =  ast.nextSolidToken();
                if (st1 != null){
                    if (st1.tokencode == '('){

                    }else{
                        ast.tokencode = TBaseType.ident; // change trim keyword to identifier if not followed by (), trim()
                    }
                }
            }

            if (dbVendor == EDbVendor.dbvopenedge){
                if (ast.tokencode == TBaseType.rrw_order){
                        TSourceToken st1 = ast.searchToken(TBaseType.rrw_by,1);
                        if (st1 == null) {
                            ast.tokencode = TBaseType.ident;
                        }
                }else if (ast.tokencode == TBaseType.rrw_with){
                        TSourceToken st1 = ast.searchToken(TBaseType.rrw_check,1);
                        if (st1 != null) {
                            ast.tokencode = TBaseType.rrw_openedge_with_check;
                        }
                }
            }

            if ( gst == EFindSqlStateType.ststoredprocedurebody )
            {
                if ( !(
                        (ast.tokencode == TBaseType.rrw_go)
                                || (ast.tokencode == TBaseType.rrw_create)
                                || (ast.tokencode == TBaseType.rrw_alter) )
                        )
                {
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                    continue;
                }
            }

            TCustomSqlStatement lcnextsqlstatement =  sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

            switch(gst){
                case sterror:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        gst = EFindSqlStateType.stsql;
                    }
                    else if ( (ast.tokentype == ETokenType.ttsemicolon) )
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                }
                case stnormal :
                {
                    if ( (ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if ( TBaseType.assigned(gcurrentsqlstatement) )
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        if ( TBaseType.assigned(lcprevsolidtoken) && (ast.tokentype == ETokenType.ttsemicolon) )
                        {
                            if ( lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                            {
                                // ;;;; continuous semicolon,treat it as comment
                                ast.tokentype = ETokenType.ttsimplecomment;
                                ast.tokencode =  TBaseType.cmtdoublehyphen;
                            }
                            else
                            {
                            }

                        }

                        continue;
                    }

                    gcurrentsqlstatement = lcnextsqlstatement; //isstoredprocedure(ast,dbvendor,gst,sqlstatements);

                    if ( TBaseType.assigned(gcurrentsqlstatement) )
                    {
                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                            //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case   sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstmssqlbegintry:
                            case sstmssqlbegincatch:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.sttrycatch;
                                lctrycatchlevel = 0;
                                break;
                            }
                            case sstmssqlgo:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            default:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stsql;
                                break;
                            }
                        }    // case
                    }
                    else
                    {
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stblock;
                        }
                        else
                        {
                            if ( sqlstatements.size() == 0 )
                            {
                                //first statement of mssql batch, treat it as exec sp
                                gst = EFindSqlStateType.stsql;
                                gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                                //tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
// todo need previous line need to be implemented
                                gcurrentsqlstatement.addtokentolist(ast);
                            }
                            else if ( sqlstatements.get(sqlstatements.size() -1).sqlstatementtype == ESqlStatementType.sstmssqlgo )
                            {
                                // prev sql is go, treat it as exec sp
                                gst = EFindSqlStateType.stsql;
                                gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                                //todo need to be implemented: tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
                                gcurrentsqlstatement.addtokentolist(ast);
                            }
                            else
                            {
                            }
                        }
                    }


                    if (  !TBaseType.assigned(gcurrentsqlstatement)  ) //error tokentext found
                    {


                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));


                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    break;
                }
                case stblock:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else
                        {
                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.stblock )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            // { [distributed] transaxtion/trans statement
                            // { dialog [ conversation ]
                            // { conversation timer
                            //  doesn't start block ({ .. })
                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( ! (TBaseType.mysametext(lcnextsolidtoken.astext,"tran")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"transaction")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"distributed")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"dialog")
                                        || TBaseType.mysametext(lcnextsolidtoken.astext,"conversation")
                                ) )
                                    lcblocklevel++;
                            }
                            else
                                lcblocklevel++;

                        }
                        else if ( ast.tokencode == TBaseType.rrw_case )  // case ... }
                            lcblocklevel++;
                        else if ( ast.tokencode == TBaseType.rrw_end )
                        {

                            lcisendconversation = false;


                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( lcnextsolidtoken.tokencode == flexer.getkeywordvalue("conversation".toUpperCase()))
                                    lcisendconversation = true; // } conversation statement
                            }


                            if ( ! lcisendconversation )
                            {

                                if ( lcblocklevel == 0 )
                                {
                                    if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                                    {
                                        if ( TBaseType.assigned(lcnextsolidtoken) )
                                        {
                                            if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                            {
                                                // { .. } else
                                                gst = EFindSqlStateType.stsql;
                                            }
                                            else if ( lcnextsolidtoken.tokentype == ETokenType.ttsemicolon )
                                            {
                                                lcnnextsolidtoken = sourcetokenlist.nextsolidtoken(lcnextsolidtoken.posinlist,1,false);
                                                if ( TBaseType.assigned(lcnnextsolidtoken) )
                                                {
                                                    if ( lcnnextsolidtoken.tokencode == TBaseType.rrw_else )
                                                    {
                                                        // { .. } else
                                                        gst = EFindSqlStateType.stsql;
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    if ( gst != EFindSqlStateType.stsql )
                                    {
                                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                                        gst = EFindSqlStateType.stnormal;
                                    }

                                }
                                else
                                {
                                    lcblocklevel--;
                                }

                            }
                        }
                    }
                    break;
                }
                case sttrycatch:
                {

                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else
                        {
                            if (
                                    (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry)
                                            ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                                    )
                                lctrycatchlevel++;
                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.sttrycatch )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( (ast.tokencode == TBaseType.rrw_try) ||
                                (ast.tokencode == TBaseType.rrw_catch) )
                        {
                            // lcprevsolidtoken = sourcetokenlist.solidtokenbefore(i);
                            if ( TBaseType.assigned(lcprevsolidtoken) )
                            {
                                if ( lcprevsolidtoken.tokencode == TBaseType.rrw_end )
                                {
                                    if ( lctrycatchlevel == 0 )
                                    {
                                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                                        gst = EFindSqlStateType.stnormal;
                                    }
                                    else
                                        lctrycatchlevel--;
                                }
                            }
                        }
                    }
                    break;
                }
                case stprocedureWithReturn:
                {
                    // found return statement in create procedure/function, which means can be ended with a semicolon

                    if ( TBaseType.assigned(lcnextsqlstatement) ) {
                        if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo) {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                            break;
                        }
                    }

                    gcurrentsqlstatement.addtokentolist(ast);
                    if ((gst == EFindSqlStateType.stprocedureWithReturn) && (ast.tokentype == ETokenType.ttsemicolon)){

                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    break;
                }
                case stsql :
                {
                    if ( (ast.tokentype == ETokenType.ttsemicolon)   )
                    {
                        lcstillinsql = false;
                        if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                        {
                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                            if ( TBaseType.assigned(lcnextsolidtoken) )
                            {
                                if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                {
                                    // if ( expr stmt; else
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    lcstillinsql = true;
                                }

                            }
                        }

                        if (  !lcstillinsql )
                        {
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gcurrentsqlstatement.semicolonended = ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }

                    }
                    else if ( TBaseType.assigned(lcnextsqlstatement) )
                    {

                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                            continue;
                        }

                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlif:
                            case sstmssqlwhile:
                            {
                                if ((lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
                                        ||(lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry))
                                {
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    gst = EFindSqlStateType.stblock;
                                    lcblocklevel = 1;
                                    lcnextsqlstatement = null;
                                    continue;

                                }
                                // if ( || while only contain one sql(not closed by {/} pair)
                                // so will still in if ( || while statement
                                else if ( gcurrentsqlstatement.dummytag == 1 )
                                {
                                    // if ( cond ^stmt nextstmt (^ stands for current pos)
                                    gcurrentsqlstatement.addtokentolist(ast);

                                    if ( (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                                            )
                                        gcurrentsqlstatement.dummytag = 1;
                                    else
                                        gcurrentsqlstatement.dummytag = 0;


                                    lcnextsqlstatement = null;
                                    continue;
                                }else{
                                    // 在 if statement 中，已经遇见过第一个sql语句，现在是碰到第二个，正常情况下这个第二个
                                    //IF EXISTS (SELECT * FROM sysobjects WHERE id = OBJECT_ID(N'[DF_TEST]') AND type = 'D')
                                    //ALTER TABLE test DROP CONSTRAINT IF EXISTS DF_TEST;

                                    if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif){
                                        // 上例中的第二个 IF EXISTS 不是真正的 sql， 忽略它
                                        if ((ast.nextSolidToken() != null)&&(ast.nextSolidToken().tokencode == TBaseType.rrw_sqlserver_exists)){
                                            gcurrentsqlstatement.addtokentolist(ast);
                                            lcnextsqlstatement = null;
                                            continue;
                                        }
                                    }
                                }
                                break;
                            }//
                            case sstmssqlalterqueue:
                            {
                                if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlexec )
                                {
                                    // execute can't be used to delimite alter queue
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    lcnextsqlstatement = null;
                                    continue;

                                }
                                break;
                            }
                            case sstmssqlcreateschema:
                            {
                                gcurrentsqlstatement.addtokentolist(ast);
                                lcnextsqlstatement = null;
                                continue;
                            }
                        }//case

                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);

                        switch(gcurrentsqlstatement.sqlstatementtype){    //
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                            //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstmssqlbegintry:
                            case sstmssqlbegincatch:
                            {
                                gst = EFindSqlStateType.sttrycatch;
                                lctrycatchlevel = 0;
                                break;
                            }
                            case sstmssqlgo:
                            {
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            default:
                            {
                                gst = EFindSqlStateType.stsql;
                                break;
                            }
                        }    // case

                    }//TBaseType.assigned(lcnextsqlstatement)
                    else if ( (ast.tokencode == TBaseType.rrw_begin) )
                    {
                        if ( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                                )
                        {
                            // start block of if ( || while statement
                            gst = EFindSqlStateType.stblock;
                            lcblocklevel = 0;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                        else if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqldeclare )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stblock;
                        }
                        else
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_case) ){
                        case_end_nest++;
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_end) ){
                        if (case_end_nest > 0){
                            case_end_nest--;
                        }
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    else if ( (ast.tokencode == TBaseType.rrw_else) )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        // if ( cond stmt ^else stmt
                        if (( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                        ) && (case_end_nest == 0))
                            gcurrentsqlstatement.dummytag = 1; // reduce to 1 while stmt after else is found: if ( cond stmt ^else stmt
                    }
                    else
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                    }
                    break;
                }
                case ststoredprocedure:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gcurrentsqlstatement = lcnextsqlstatement;
                            gcurrentsqlstatement.addtokentolist(ast);
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            gst = EFindSqlStateType.stnormal;
                        }
                        else if (lcnextsqlstatement.sqlstatementtype == sstmssqlreturn)
                        {
                            gst = EFindSqlStateType.stprocedureWithReturn;
                            gcurrentsqlstatement.addtokentolist(ast);
                            lcnextsqlstatement = null;
                        }
                        else
                        {
                            gst = EFindSqlStateType.ststoredprocedurebody;
                            gcurrentsqlstatement.addtokentolist(ast);

                            lcnextsqlstatement = null;
                        }
                    }

                    if ( gst == EFindSqlStateType.ststoredprocedure )
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ( ast.tokencode == TBaseType.rrw_begin )
                        {
                            gst = EFindSqlStateType.stblock;
                        }
                    }
                    break;
                } //stplsql
                case ststoredprocedurebody:
                {
                    if ( TBaseType.assigned(lcnextsqlstatement) )
                    {
                        switch(lcnextsqlstatement.sqlstatementtype){    //
                            case sstmssqlgo:
                            {
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gst = EFindSqlStateType.stnormal;
                                break;
                            }
                            case sstmssqlcreateprocedure:
                            case sstmssqlcreatefunction:
                            //case sstmssqlcreatetrigger:
                            case sstcreatetrigger:
                            case sstmssqlalterprocedure:
                            case sstmssqlalterfunction:
                            case sstmssqlaltertrigger:
                            {
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.ststoredprocedure;
                                break;
                            }
                            case sstcreateview:
                            case sstcreatetable:
                            {

                                boolean readForNewStmt = false;
                                TSourceToken st1 = ast.searchToken(';',-1);
                                if (st1 != null) {
                                    TSourceToken st2 = ast.searchToken(TBaseType.rrw_end,-2);
                                    if (st2 != null){
                                        readForNewStmt = true;
                                    }
                                }

                                if (readForNewStmt){
                                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                                    gcurrentsqlstatement = lcnextsqlstatement;
                                    gcurrentsqlstatement.addtokentolist(ast);
                                    gst = EFindSqlStateType.stsql;
                                }else{
                                    lcnextsqlstatement = null;
                                }
                                break;
                            }
                            case sstmssqlDropSecurityPolicy:
                            case sstmssqlAlterSecurityPolicy:
                            case sstmssqlCreateSecurityPolicy:
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                gcurrentsqlstatement = lcnextsqlstatement;
                                gcurrentsqlstatement.addtokentolist(ast);
                                gst = EFindSqlStateType.stsql;

                                    break;
                            default:
                            {
                                lcnextsqlstatement = null;
                                break;
                            }
                        }//case
                    }

                    if ( gst == EFindSqlStateType.ststoredprocedurebody )
                        gcurrentsqlstatement.addtokentolist(ast);
                }
                break;
            } //case
        } //for


        //last statement
        if ( TBaseType.assigned(gcurrentsqlstatement) &&  (gst != EFindSqlStateType.stnormal))
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }



        return errorcount;

    }


    private TCustomSqlStatement startDaxStmt(TSourceToken currToken,TCustomSqlStatement currStmt){
        TCustomSqlStatement newStmt = null;
        if (currToken == null) return null;
        if ((currToken.tokencode == '=')&&(currToken.isFirstTokenOfLine())){
            currToken.tokencode = TBaseType.equal_start_expr;
            newStmt = new TDaxExprStmt(EDbVendor.dbvdax);
        }else if ((currToken.tokencode == TBaseType.rrw_dax_define)&&(currToken.isFirstTokenOfLine())){
            newStmt = new TDaxEvaluateStmt(EDbVendor.dbvdax);
            ((TDaxEvaluateStmt)newStmt).setStartWithDefine(true);
        }else if ((currToken.tokencode == TBaseType.rrw_dax_evaluate)&&(currToken.isFirstTokenOfLine())){
            if ((currStmt != null)&&(currStmt instanceof TDaxEvaluateStmt)){
                TDaxEvaluateStmt tmp = (TDaxEvaluateStmt)currStmt;
                if (tmp.isStartWithDefine()) return  null;
            }
            newStmt = new TDaxEvaluateStmt(EDbVendor.dbvdax);
        }

        if (newStmt == null){
            // let's check is this the first token of query
            boolean isFirst = currToken.isFirstTokenOfLine();
            TSourceToken prevToken = currToken.prevSolidToken();
            if ((isFirst)&&(prevToken == null)){
                newStmt = new TDaxExprStmt(EDbVendor.dbvdax);
            }
        }
        return newStmt;
    }

    int doodbcgetrawsqlstatements(){
        return domssqlgetrawsqlstatements();
    }

    int dodaxgetrawsqlstatements(){
        gcurrentsqlstatement = null;
        TCustomSqlStatement tmpStmt = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        int i;
        TSourceToken ast;
        int errorcount = 0;
        int nestedParens = 0;

        for (i=0 ; i<sourcetokenlist.size();i++)
        {
            ast = sourcetokenlist.get(i);
            //System.out.println(ast);
            sourcetokenlist.curpos = i;

            if ((ast.tokencode == TBaseType.rrw_dax_product)
                    ||(ast.tokencode == TBaseType.rrw_dax_true)
                    ||(ast.tokencode == TBaseType.rrw_dax_false)
                    ||(ast.tokencode == TBaseType.rrw_dax_calendar)
                    ||(ast.tokencode == TBaseType.rrw_dax_second)
                    ){
                TSourceToken st1 = ast.searchToken("(",1);
                if (st1 == null){
                    ast.tokencode = TBaseType.ident;
                }
            }

            if (ast.tokencode == '(') nestedParens++;
            if (ast.tokencode == ')') nestedParens--;
            if (nestedParens > 0){
                if (gcurrentsqlstatement != null){
                    gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                continue;
            }

            switch(gst){
                case sterror:
                {
                    tmpStmt = startDaxStmt(ast,gcurrentsqlstatement);
                    if (tmpStmt != null){
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        gcurrentsqlstatement =  tmpStmt;
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        gst = EFindSqlStateType.stsql;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                }
                case stnormal :
                {
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon))
                    {
                        if (TBaseType.assigned(gcurrentsqlstatement))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                        continue;
                    }

                    gcurrentsqlstatement = startDaxStmt(ast,gcurrentsqlstatement);

                    if (TBaseType.assigned(gcurrentsqlstatement))
                    {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                    }else
                    {

//                        errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                        errorcount = 1;
                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }
                    break;
                }
                case stsql :
                {
                    tmpStmt = startDaxStmt(ast,gcurrentsqlstatement);
                    if (tmpStmt != null){
                        gst = EFindSqlStateType.stsql;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        gcurrentsqlstatement = tmpStmt;
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    gcurrentsqlstatement.addtokentolist(ast);

                    break;
                }

            } //case
        } //for


        //last statement
        if (TBaseType.assigned(gcurrentsqlstatement) && ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.sterror)) )
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return errorcount;//FSqlStatements.Count;

    }


 int dohanagetrawsqlstatements(){
    int errorcount = 0;
    int case_end_nest = 0;

    if ( TBaseType.assigned(sqlstatements) ) sqlstatements.clear();
    if ( ! TBaseType.assigned(sourcetokenlist) ) return -1;

    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    int lcblocklevel = 0;
    int lctrycatchlevel = 0;
    TSourceToken lcprevsolidtoken = null,lcnextsolidtoken,lcnnextsolidtoken;
    TSourceToken ast = null;
    int i,lcMergeInSelectNested = 0;
    boolean lcisendconversation, lcstillinsql,lcMergeInSelect = false;

    for (i=0 ; i < sourcetokenlist.size();i++)
      {

          if ( (ast != null ) && (ast.issolidtoken() ))
            lcprevsolidtoken = ast;

        ast = sourcetokenlist.get(i);
        sourcetokenlist.curpos = i;

        if (lcMergeInSelect){
            if (ast.tokencode == '(') lcMergeInSelectNested++;
            if (ast.tokencode == ')'){
                lcMergeInSelectNested--;
                if (lcMergeInSelectNested == 0){
                    lcMergeInSelect = false;
                }
            }
            gcurrentsqlstatement.sourcetokenlist.add(ast);
            continue;
        }
        if ( ast.tokenstatus == ETokenStatus.tsignoredbygetrawstatement )
          {
            //tsignoredbygetrawstatement is set when cte is found in dbcmds.findcte function
            gcurrentsqlstatement.sourcetokenlist.add(ast);
            continue;
          }

        if (ast.tokencode == TBaseType.rrw_minus){
            TSourceToken st1 = ast.searchToken('(',1);
            if (st1 == null){
                st1 = ast.searchToken(TBaseType.rrw_select,1);
                if (st1 == null){
                    ast.tokencode = TBaseType.ident;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_merge){
            TSourceToken st1 = ast.nextSolidToken();
            if (st1.tokencode == TBaseType.rrw_join){
                ast.tokencode = TBaseType.rrw_merge2_sqlserver;
            }
            if ((lcprevsolidtoken != null)&&(lcprevsolidtoken.tokencode == '(')){
                lcMergeInSelect = true;
                lcMergeInSelectNested++;
                gcurrentsqlstatement.addtokentolist(ast);
                continue;
            }
        }else if (ast.tokencode == TBaseType.rrw_as){
            TSourceToken st1 = ast.nextSolidToken();
            if (st1.tokencode == TBaseType.rrw_hana_of){
                ast.tokencode = TBaseType.rrw_as_before_of;
            }
        }else if (ast.tokencode == TBaseType.rrw_date){
            TSourceToken st1 = ast.nextSolidToken();
            if (st1.tokencode == TBaseType.sconst){
                ast.tokencode = TBaseType.rrw_hana_date_const;
            }
        }else if (ast.tokencode == TBaseType.rrw_time){
            TSourceToken st1 = ast.nextSolidToken();
            if (st1.tokencode == TBaseType.sconst){
                ast.tokencode = TBaseType.rrw_hana_time_const;
            }
        }else if (ast.tokencode == TBaseType.rrw_timestamp){
            TSourceToken st1 = ast.nextSolidToken();
            if (st1.tokencode == TBaseType.sconst){
                ast.tokencode = TBaseType.rrw_hana_timestamp_const;
            }
        }else if (ast.tokencode == TBaseType.rrw_with){
            TSourceToken st1 = ast.nextSolidToken();
            if (st1.toString().equalsIgnoreCase("structured")){
                ast.tokencode = TBaseType.rrw_hana_with_structured;
            }else if (st1.toString().equalsIgnoreCase("cache")){
                ast.tokencode = TBaseType.rrw_hana_with_cache;
            }else if (st1.toString().equalsIgnoreCase("static")){
                ast.tokencode = TBaseType.rrw_hana_with_cache;
            }else if (st1.toString().equalsIgnoreCase("dynamic")){
                ast.tokencode = TBaseType.rrw_hana_with_cache;
            }else if (st1.toString().equalsIgnoreCase("check")){
                ast.tokencode = TBaseType.rrw_hana_with_check;
            }else if (st1.toString().equalsIgnoreCase("mask")){
                ast.tokencode = TBaseType.rrw_hana_with_mask;
            }else if (st1.toString().equalsIgnoreCase("expression")){
                ast.tokencode = TBaseType.rrw_hana_with_expression;
            }else if (st1.toString().equalsIgnoreCase("anonymization")){
                ast.tokencode = TBaseType.rrw_hana_with_anonymization;
            }else if (st1.toString().equalsIgnoreCase("hint")){
                ast.tokencode = TBaseType.rrw_hana_with_hint;
            }
        }else if (ast.tokencode == TBaseType.rrw_hana_unload){
            TSourceToken st1 = ast.nextSolidToken();
            if (st1.toString().equalsIgnoreCase("priority")){
                ast.tokencode = TBaseType.rrw_hana_unload2;
            }
        }

        if ( gst == EFindSqlStateType.ststoredprocedurebody )
          {
            if ( !(
                (ast.tokencode == TBaseType.rrw_go)
                || (ast.tokencode == TBaseType.rrw_create)
                || (ast.tokencode == TBaseType.rrw_alter) )
              )
              {
                gcurrentsqlstatement.sourcetokenlist.add(ast);
                continue;
              }
          }

        TCustomSqlStatement lcnextsqlstatement =  sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

        switch(gst){
          case sterror:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gcurrentsqlstatement = lcnextsqlstatement;
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  gst = EFindSqlStateType.stsql;
                }
              else if ( (ast.tokentype == ETokenType.ttsemicolon) )
                {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
                }
              else
                {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                }
                break;
            }
          case stnormal :
            {
              if ( (ast.tokencode  == TBaseType.cmtdoublehyphen)
                 || (ast.tokencode  == TBaseType.cmtslashstar)
                 || (ast.tokencode  == TBaseType.lexspace)
                 || (ast.tokencode  == TBaseType.lexnewline)
                 || (ast.tokentype  == ETokenType.ttsemicolon) )
                {
                 if ( TBaseType.assigned(gcurrentsqlstatement) )
                   {
                     gcurrentsqlstatement.addtokentolist(ast);
                   }

                 if ( TBaseType.assigned(lcprevsolidtoken) && (ast.tokentype == ETokenType.ttsemicolon) )
                   {
                     if ( lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                       {
                        // ;;;; continuous semicolon,treat it as comment
                         ast.tokentype = ETokenType.ttsimplecomment;
                         ast.tokencode =  TBaseType.cmtdoublehyphen;
                       }
                     else
                       {
                       }

                   }

                 continue;
                }

              gcurrentsqlstatement = lcnextsqlstatement; //isstoredprocedure(ast,dbvendor,gst,sqlstatements);

              if ( TBaseType.assigned(gcurrentsqlstatement) )
                {
                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstcreateprocedure:
                    case sstcreatefunction:
                    case sstcreatetrigger:
                   // case sstalterprocedure:
                   // case sstalterfunction:
                    case sstaltertrigger:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                      }
                    case sstmssqlgo:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                          break;
                      }
                    default:
                      {
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.stsql;
                          break;
                      }
                  }    // case
                }
              else
                {
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      gcurrentsqlstatement = new TMssqlBlock(dbVendor);
                      gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
                      gcurrentsqlstatement.addtokentolist(ast);
                      gst = EFindSqlStateType.stblock;
                    }
                  else
                    {
                      if ( sqlstatements.size() == 0 )
                        {
                          //first statement of mssql batch, treat it as exec sp
                          gst = EFindSqlStateType.stsql;
                          gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                          //tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
// todo need previous line need to be implemented
                          gcurrentsqlstatement.addtokentolist(ast);
                        }
                      else if ( sqlstatements.get(sqlstatements.size() -1).sqlstatementtype == ESqlStatementType.sstmssqlgo )
                        {
                          // prev sql is go, treat it as exec sp
                          gst = EFindSqlStateType.stsql;
                          gcurrentsqlstatement = new TMssqlExecute(dbVendor);
                          //todo need to be implemented: tmssqlexecute(gcurrentsqlstatement).exectype = metnoexeckeyword;
                          gcurrentsqlstatement.addtokentolist(ast);
                        }
                      else
                        {
                        }
                    }
                }


              if (  !TBaseType.assigned(gcurrentsqlstatement)  ) //error tokentext found
                {

//                  errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                  errorcount = 1;
                    this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                            ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                  ast.tokentype = ETokenType.tttokenlizererrortoken;
                  gst = EFindSqlStateType.sterror;

                  gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                  gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                  gcurrentsqlstatement.addtokentolist(ast);
                }
                break;
            }
          case stblock:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                  else
                    {
                      lcnextsqlstatement = null;
                    }
                }

                if (( lcblocklevel == -1 )&&(ast.tokencode==';'))
                {
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                    gst = EFindSqlStateType.stnormal;
                    break;
                }

              if ( gst == EFindSqlStateType.stblock )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      // { [distributed] transaxtion/trans statement
                      // { dialog [ conversation ]
                      // { conversation timer
                      //  doesn't start block ({ .. })
                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( ! (TBaseType.mysametext(lcnextsolidtoken.astext,"tran")
                                  || TBaseType.mysametext(lcnextsolidtoken.astext,"transaction")
                                  ) )
                           lcblocklevel++;
                        }
                      else
                        lcblocklevel++;

                    }
                  else if ( ast.tokencode == TBaseType.rrw_case )  // case ... }
                    lcblocklevel++;
                  else if ( ast.tokencode == TBaseType.rrw_end )
                    {

                      lcisendconversation = false;


                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);

                      if ( ! lcisendconversation )
                        {

                          if ( lcblocklevel == 0 )
                            {
                              if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                                {
                                  if ( TBaseType.assigned(lcnextsolidtoken) )
                                    {
                                      if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                                        {
                                          // { .. } else
                                          gst = EFindSqlStateType.stsql;
                                        }
                                      else if ( lcnextsolidtoken.tokentype == ETokenType.ttsemicolon )
                                        {
                                          lcnnextsolidtoken = sourcetokenlist.nextsolidtoken(lcnextsolidtoken.posinlist,1,false);
                                          if ( TBaseType.assigned(lcnnextsolidtoken) )
                                            {
                                              if ( lcnnextsolidtoken.tokencode == TBaseType.rrw_else )
                                                {
                                                  // { .. } else
                                                  gst = EFindSqlStateType.stsql;
                                                }
                                            }
                                        }
                                    }
                                }

//                              if ( gst != EFindSqlStateType.stsql )
//                                {
//                                  doongetrawsqlstatementevent(gcurrentsqlstatement);
//                                  gst = EFindSqlStateType.stnormal;
//                                }

                            }
                          else
                            {
                              lcblocklevel--;
                            }

                        }
                    }
                }
                break;
            }
          case stsql :
            {
              if ( (ast.tokentype == ETokenType.ttsemicolon)   )
                {
                  lcstillinsql = false;
                  if ( gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif )
                    {
                      lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i,1,false);
                      if ( TBaseType.assigned(lcnextsolidtoken) )
                        {
                          if ( lcnextsolidtoken.tokencode == TBaseType.rrw_else )
                            {
                              // if ( expr stmt; else
                              gcurrentsqlstatement.addtokentolist(ast);
                              lcstillinsql = true;
                            }

                        }
                    }

                  if (  !lcstillinsql )
                    {
                      gst = EFindSqlStateType.stnormal;
                      gcurrentsqlstatement.addtokentolist(ast);
                      gcurrentsqlstatement.semicolonended = ast;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }

                }
              else if ( TBaseType.assigned(lcnextsqlstatement) )
                {

                  if ( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                        continue;
                    }

                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstmssqlif:
                    case sstmssqlwhile:
                      {
                        // if ( || while only contain one sql(not closed by {/} pair)
                        // so will still in if ( || while statement
                        if ( gcurrentsqlstatement.dummytag == 1 )
                          {
                            // if ( cond ^stmt nextstmt (^ stands for current pos)
                            gcurrentsqlstatement.addtokentolist(ast);

                            if ( (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                              )
                                gcurrentsqlstatement.dummytag = 1;
                              else
                                gcurrentsqlstatement.dummytag = 0;


                            lcnextsqlstatement = null;
                            continue;
                          }
                      break;
                      }//
                    case sstcreateschema:
                      {
                            gcurrentsqlstatement.addtokentolist(ast);
                            lcnextsqlstatement = null;
                            continue;                          
                      }
                  }//case

                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gcurrentsqlstatement = lcnextsqlstatement;
                  gcurrentsqlstatement.addtokentolist(ast);

                  switch(gcurrentsqlstatement.sqlstatementtype){    //
                    case sstcreateprocedure:
                    case sstcreatefunction:
                    case sstcreatetrigger:
                    case sstalterprocedure:
                    case sstalterfunction:
                    case sstaltertrigger:
                      {
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                      }
                    default:
                      {
                        gst = EFindSqlStateType.stsql;
                        break;
                      }
                  }    // case

                }//TBaseType.assigned(lcnextsqlstatement)
              else if ( (ast.tokencode == TBaseType.rrw_begin) )
                {
                  if ( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                  || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                  )
                    {
                      // start block of if ( || while statement
                      gst = EFindSqlStateType.stblock;
                      lcblocklevel = 0;
                      gcurrentsqlstatement.addtokentolist(ast);
                    }
                  else
                    {
                      gcurrentsqlstatement.addtokentolist(ast);
                    }
                }
              else if ( (ast.tokencode == TBaseType.rrw_case) ){
                  case_end_nest++;
                  gcurrentsqlstatement.addtokentolist(ast);
              }
              else if ( (ast.tokencode == TBaseType.rrw_end) ){
                  if (case_end_nest > 0){
                      case_end_nest--;
                  }
                  gcurrentsqlstatement.addtokentolist(ast);
              }
              else if ( (ast.tokencode == TBaseType.rrw_else) )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  // if ( cond stmt ^else stmt
                  if (( (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
                  || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
                  ) && (case_end_nest == 0))
                    gcurrentsqlstatement.dummytag = 1; // reduce to 1 while stmt after else is found: if ( cond stmt ^else stmt
                }
              else
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                }
             break;
            }
          case ststoredprocedure:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {


                  if (( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstalterfunction )
                          ||( lcnextsqlstatement.sqlstatementtype == sstcreateprocedure )
                          ||( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatefunction )
                          ||( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatetrigger )
                          ||( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstalterprocedure )
                          ||( lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstaltertrigger )
                          )
                    {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                    }
                  else
                    {
                      gst = EFindSqlStateType.ststoredprocedurebody;
                      gcurrentsqlstatement.addtokentolist(ast);

                      lcnextsqlstatement = null;
                    }
                }

              if ( gst == EFindSqlStateType.ststoredprocedure )
                {
                  gcurrentsqlstatement.addtokentolist(ast);
                  if ( ast.tokencode == TBaseType.rrw_begin )
                    {
                      gst = EFindSqlStateType.ststoredprocedurebody;
                    }
                    else if ( ast.tokencode == TBaseType.rrw_hana_header )
                    {
                        gst = EFindSqlStateType.ststoredprocedurebody;
                    }
                }
             break;
            } //stplsql
          case ststoredprocedurebody:
            {
              if ( TBaseType.assigned(lcnextsqlstatement) )
                {
                  switch(lcnextsqlstatement.sqlstatementtype){    //
                    case sstmssqlgo:
                      {
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gcurrentsqlstatement = lcnextsqlstatement;
                          gcurrentsqlstatement.addtokentolist(ast);
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gst = EFindSqlStateType.stnormal;
                          break;
                      }
                    case sstcreateprocedure:
                    case sstcreatefunction:
                    case sstcreatetrigger:
                    case sstalterprocedure:
                    case sstalterfunction:
                    case sstaltertrigger:
                      {
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gcurrentsqlstatement = lcnextsqlstatement;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gst = EFindSqlStateType.ststoredprocedure;
                        break;
                      }
                    default:
                      {
                        lcnextsqlstatement = null;
                          break;
                      }
                  }//case
                }

              if ( gst == EFindSqlStateType.ststoredprocedurebody )
                gcurrentsqlstatement.addtokentolist(ast);
            }
          break;
        } //case
      } //for


    //last statement
    if ( TBaseType.assigned(gcurrentsqlstatement) &&  (gst != EFindSqlStateType.stnormal))
      {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
      }

    return errorcount;

}

int dosparksqlgetrawsqlstatements(){


        int errorcount = 0;
        gcurrentsqlstatement = null;
        EFindSqlStateType gst = EFindSqlStateType.stnormal;
        int i,c;
        TSourceToken ast;
        //char ch;
        // String lcstr,delimterTokenStr;
        boolean waitingDelimiter = false;

        //reset delimiter
        userDelimiterStr = defaultDelimiterStr;

        for (i=0;i < sourcetokenlist.size();i++)
        {
            ast = sourcetokenlist.get(i);
            sourcetokenlist.curpos = i;

            if (ast.tokencode == TBaseType.rrw_date){
                TSourceToken st1 =  ast.nextSolidToken(); //ast.searchToken('(',1);
                if (st1 != null){
                    if (st1.tokencode == '('){
                        ast.tokencode = TBaseType.rrw_spark_date_function;
                    }else if (st1.tokencode == TBaseType.sconst){
                        ast.tokencode = TBaseType.rrw_spark_date_const;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_time){
                TSourceToken st1 =  ast.nextSolidToken();
                if (st1 != null){
                    if (st1.tokencode == TBaseType.sconst){
                        ast.tokencode = TBaseType.rrw_spark_time_const;
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_timestamp){
                TSourceToken st1 =  ast.nextSolidToken();
                if (st1 != null){
                    if (st1.tokencode == TBaseType.sconst){
                        ast.tokencode = TBaseType.rrw_spark_timestamp_constant;
                    }else if (st1.tokencode == TBaseType.ident){
                        if (st1.toString().startsWith("\"")){
                            ast.tokencode = TBaseType.rrw_spark_timestamp_constant;
                            st1.tokencode = TBaseType.sconst;
                        }
                    }
                }
            }
            else if (ast.tokencode == TBaseType.rrw_interval){
                TSourceToken leftParen = ast.searchToken('(',1);
                if ( leftParen != null){
                    int k = leftParen.posinlist + 1;
                    boolean commaToken = false;
                    while (k<ast.container.size()){
                        if (ast.container.get(k).tokencode == ')') break;
                        if (ast.container.get(k).tokencode == ','){
                            commaToken = true;
                            break;
                        }
                        k++;
                    }
                    if (commaToken){
                        ast.tokencode = TBaseType.rrw_mysql_interval_func;
                    }
                }
            }

            switch(gst){
                case sterror:
                {
                    if (ast.tokentype == ETokenType.ttsemicolon)
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        gst = EFindSqlStateType.stnormal;
                    }
                    else
                    {
                        gcurrentsqlstatement.sourcetokenlist.add(ast);
                    }
                    break;
                }
                case stnormal :
                {
                    if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                            || (ast.tokencode  == TBaseType.cmtslashstar)
                            || (ast.tokencode  == TBaseType.lexspace)
                            || (ast.tokencode  == TBaseType.lexnewline)
                            || (ast.tokentype  == ETokenType.ttsemicolon) )
                    {
                        if (TBaseType.assigned(gcurrentsqlstatement))
                        {
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                        continue;
                    }

                    if ((ast.isFirstTokenOfLine())&&((ast.tokencode == TBaseType.rrw_mysql_source ) || (ast.tokencode == TBaseType.slash_dot)))
                    {
                        gst = EFindSqlStateType.stsqlplus;
                        gcurrentsqlstatement = new TMySQLSource(dbVendor);
                        gcurrentsqlstatement.addtokentolist(ast);
                        continue;
                    }

                    // find a tokentext to start sql or plsql mode
                    gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

                    if (TBaseType.assigned(gcurrentsqlstatement))
                    {
                        ESqlStatementType[] ses = {ESqlStatementType.sstmysqlcreateprocedure, ESqlStatementType.sstmysqlcreatefunction
                                , sstcreateprocedure, ESqlStatementType.sstcreatefunction
                                , ESqlStatementType.sstcreatetrigger};
                        if(includesqlstatementtype(gcurrentsqlstatement.sqlstatementtype, ses) )
                        {
                            gst = EFindSqlStateType.ststoredprocedure;
                            waitingDelimiter = false;
                            gcurrentsqlstatement.addtokentolist(ast);
                            curdelimiterchar = ';';
//                        if (userDelimiterStr == ""){
//                            userDelimiterStr = defaultDelimiterStr;
//                        }
                        }
                        else
                        {
                            gst = EFindSqlStateType.stsql;
                            gcurrentsqlstatement.addtokentolist(ast);
                        }

                    }


                    if (!TBaseType.assigned(gcurrentsqlstatement) ) //error tokentext found
                    {

//                  errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                  errorcount = 1;
                        this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                                ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                        ast.tokentype = ETokenType.tttokenlizererrortoken;
                        gst = EFindSqlStateType.sterror;

                        gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                        gcurrentsqlstatement.addtokentolist(ast);

                    }
                    break;
                }
                case stsqlplus:{
                    if (ast.tokencode  == TBaseType.lexnewline){
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast); // so add it here
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                    }else {
                        {gcurrentsqlstatement.addtokentolist(ast);}
                    }

                    break;
                }//case source command or \. command
                case stsql :
                {
                    if ((ast.tokentype == ETokenType.ttsemicolon)&&(gcurrentsqlstatement.sqlstatementtype != ESqlStatementType.sstmysqldelimiter))
                    {
                        gst = EFindSqlStateType.stnormal;
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    if (ast.toString().equalsIgnoreCase(userDelimiterStr))
                    {
                        gst = EFindSqlStateType.stnormal;
                        ast.tokencode = ';';// treat it as semicolon
                        gcurrentsqlstatement.addtokentolist(ast);
                        gcurrentsqlstatement.semicolonended = ast;
                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                        continue;
                    }
                    gcurrentsqlstatement.addtokentolist(ast);

                    if ((ast.tokencode  == TBaseType.lexnewline)
                            &&(gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmysqldelimiter)){
                        gst = EFindSqlStateType.stnormal;
                        userDelimiterStr = "";
                        for(int k=0;k<gcurrentsqlstatement.sourcetokenlist.size();k++){
                            TSourceToken st = gcurrentsqlstatement.sourcetokenlist.get(k);
                            if ((st.tokencode == TBaseType.rrw_mysql_delimiter)
                                    ||(st.tokencode == TBaseType.lexnewline)
                                    ||(st.tokencode == TBaseType.lexspace)
                                    ||(st.tokencode == TBaseType.rrw_set) // set delimiter //
                            )
                            {
                                continue;
                            }

                            userDelimiterStr += st.toString();
                            // System.out.println("userDelimiterStr:"+userDelimiterStr);
                            // System.out.print(gcurrentsqlstatement.sourcetokenlist.get(k).toString());
                        }
                        doongetrawsqlstatementevent(gcurrentsqlstatement);

                        continue;
                    }

                    break;
                }
                case ststoredprocedure :
                {
                    // single stmt in function/procedure/trigger may use ; as terminate char
                    // so default terminate char is ;, if begin is found
                    // then set terminate char to DelimiterChar
//                    if (!userDelimiterStr.equals(defaultDelimiterStr) )
//                      if (ast.tokencode == TBaseType.rrw_begin)
//                        userDelimiterStr = defaultDelimiterStr;
                    if (waitingDelimiter){
                        if (userDelimiterStr.equalsIgnoreCase(ast.toString())){
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.semicolonended = ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }else if (userDelimiterStr.startsWith(ast.toString())){
                            String lcstr = ast.toString();
                            for (int k=ast.posinlist+1;k<ast.container.size();k++) {
                                TSourceToken st = ast.container.get(k);
                                if ((st.tokencode == TBaseType.rrw_mysql_delimiter) || (st.tokencode == TBaseType.lexnewline) || (st.tokencode == TBaseType.lexspace)) {
                                    break;
                                }
                                lcstr = lcstr + st.toString();
                            }

                            if (userDelimiterStr.equalsIgnoreCase(lcstr)){
                                for (int k=ast.posinlist;k<ast.container.size();k++) {
                                    TSourceToken st = ast.container.get(k);
                                    if ((st.tokencode == TBaseType.rrw_mysql_delimiter) || (st.tokencode == TBaseType.lexnewline) || (st.tokencode == TBaseType.lexspace)) {
                                        break;
                                    }
                                    ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                                }
                                gst = EFindSqlStateType.stnormal;
                                gcurrentsqlstatement.semicolonended = ast;
                                doongetrawsqlstatementevent(gcurrentsqlstatement);
                                continue;
                            }

                        }
                    }
                    if (ast.tokencode == TBaseType.rrw_begin)
                        waitingDelimiter = true;

                    if (userDelimiterStr.equals(";")||(waitingDelimiter == false))
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if (ast.tokentype == ETokenType.ttsemicolon)
                        {
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.semicolonended = ast;
                            // asqlstatement._semicolon := ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }
                    else
                    {
                        if (ast.toString().equals(userDelimiterStr) )
                        {
                            ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                            gcurrentsqlstatement.addtokentolist(ast);
                            gst = EFindSqlStateType.stnormal;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                        }
                        else{
                            if ((ast.tokentype == ETokenType.ttsemicolon)&&(userDelimiterStr.equals(";")))
                            {
                                TSourceToken lcprevtoken = ast.container.nextsolidtoken(ast,-1,false);
                                if (lcprevtoken != null)
                                {
                                    if (lcprevtoken.tokencode == TBaseType.rrw_end)
                                    {
                                        gst = EFindSqlStateType.stnormal;
                                        gcurrentsqlstatement.semicolonended = ast;
                                        gcurrentsqlstatement.addtokentolist(ast);
                                        doongetrawsqlstatementevent(gcurrentsqlstatement);
                                        continue;
                                    }
                                }
                            }

                            gcurrentsqlstatement.addtokentolist(ast);
                        }
                    } // end of DelimiterChar
                    break;
                } //mysql sp
            } //case
        } //for


        //last statement
        if (TBaseType.assigned(gcurrentsqlstatement) && ( (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) || (gst == EFindSqlStateType.sterror))
        )
        {
            doongetrawsqlstatementevent(gcurrentsqlstatement);
        }

        return errorcount;

}

int domysqlgetrawsqlstatements(){
    int errorcount = 0;
    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    int i,c;
    TSourceToken ast;
    //char ch;
   // String lcstr,delimterTokenStr;
    boolean waitingDelimiter = false;

    //reset delimiter
    userDelimiterStr = defaultDelimiterStr;

    for (i=0;i < sourcetokenlist.size();i++)
    {
        ast = sourcetokenlist.get(i);
        sourcetokenlist.curpos = i;

        if (ast.tokencode == TBaseType.rrw_date){
            TSourceToken st1 =  ast.nextSolidToken(); //ast.searchToken('(',1);
            if (st1 != null){
                if (st1.tokencode == '('){
                    ast.tokencode = TBaseType.rrw_mysql_date_function;
                }else if (st1.tokencode == TBaseType.sconst){
                    ast.tokencode = TBaseType.rrw_mysql_date_const;
                }
            }
        }
        else if (ast.tokencode == TBaseType.rrw_time){
            TSourceToken st1 =  ast.nextSolidToken();
            if (st1 != null){
                if (st1.tokencode == TBaseType.sconst){
                    ast.tokencode = TBaseType.rrw_mysql_time_const;
                }
            }
        }
        else if (ast.tokencode == TBaseType.rrw_timestamp){
            TSourceToken st1 =  ast.nextSolidToken();
            if (st1 != null){
                if (st1.tokencode == TBaseType.sconst){
                    ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
                }else if (st1.tokencode == TBaseType.ident){
                    if (st1.toString().startsWith("\"")){
                        ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
                        st1.tokencode = TBaseType.sconst;
                    }
                }
            }
        }
        else if (ast.tokencode == TBaseType.rrw_mysql_position){
            TSourceToken st1 =  ast.nextSolidToken();
            if (st1 != null){
                if (st1.tokencode == '('){

                }else{
                    ast.tokencode = TBaseType.ident; // change position keyword to identifier if not followed by (), position()
                }
            }
        }
        else if (ast.tokencode == TBaseType.rrw_mysql_row){
            boolean isIdent = true;
            TSourceToken st1 =  ast.nextSolidToken();
            if (st1 != null){
                if (st1.tokencode == '('){
                    isIdent = false;
                }
            }
            st1 =  ast.prevSolidToken();
            if (st1 != null){
                if ((st1.tokencode == TBaseType.rrw_mysql_each)||(st1.tokencode == TBaseType.rrw_mysql_current)){
                    isIdent = false;
                }
            }
            if (isIdent) ast.tokencode = TBaseType.ident;
        }else if (ast.tokencode == TBaseType.rrw_interval){
            TSourceToken leftParen = ast.searchToken('(',1);
            if ( leftParen != null){
                int k = leftParen.posinlist + 1;
                int nested = 1;
                boolean commaToken = false;
                while (k<ast.container.size()){
                    if (ast.container.get(k).tokencode == '(') {
                        nested++;
                    }
                    if (ast.container.get(k).tokencode == ')') {
                        nested--;
                        if (nested == 0) break;
                    }
                    if ((ast.container.get(k).tokencode == ',')&&(nested == 1)){
                        // only calculate the comma in the first level which is belong to interval
                        // don't count comma in the nested () liek this: INTERVAL (SELECT IF(1=1,2,3))
                        commaToken = true;
                        break;
                    }
                    k++;
                }
                if (commaToken){
                    ast.tokencode = TBaseType.rrw_mysql_interval_func;
                }
            }
        }

        switch(gst){
          case sterror:
          {
              if (ast.tokentype == ETokenType.ttsemicolon)
              {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
              }
              else
              {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
              }
            break;
          }
          case stnormal :
          {
              if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                 || (ast.tokencode  == TBaseType.cmtslashstar)
                 || (ast.tokencode  == TBaseType.lexspace)
                 || (ast.tokencode  == TBaseType.lexnewline)
                 || (ast.tokentype  == ETokenType.ttsemicolon) )
              {
                 if (TBaseType.assigned(gcurrentsqlstatement))
                  {
                     gcurrentsqlstatement.addtokentolist(ast);
                  }

                 continue;
              }

              if (ast.isFirstTokenOfLine() && (ast.toString().equalsIgnoreCase(userDelimiterStr)))
              {
                  ast.tokencode = ';';// treat it as semicolon
                  continue;
              }

              if ((ast.isFirstTokenOfLine())&&((ast.tokencode == TBaseType.rrw_mysql_source ) || (ast.tokencode == TBaseType.slash_dot)))
              {
                  gst = EFindSqlStateType.stsqlplus;
                  gcurrentsqlstatement = new TMySQLSource(dbVendor);
                  gcurrentsqlstatement.addtokentolist(ast);
                  continue;
              }

              // find a tokentext to start sql or plsql mode
              gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

              if (TBaseType.assigned(gcurrentsqlstatement))
              {
                  ESqlStatementType[] ses = {ESqlStatementType.sstmysqlcreateprocedure, ESqlStatementType.sstmysqlcreatefunction
                          , sstcreateprocedure, ESqlStatementType.sstcreatefunction
                          , ESqlStatementType.sstcreatetrigger};
                  if(includesqlstatementtype(gcurrentsqlstatement.sqlstatementtype, ses) )
                  {
                        gst = EFindSqlStateType.ststoredprocedure;
                        waitingDelimiter = false;
                        gcurrentsqlstatement.addtokentolist(ast);
                        curdelimiterchar = ';';
//                        if (userDelimiterStr == ""){
//                            userDelimiterStr = defaultDelimiterStr;
//                        }
                  }
                  else
                  {
                      gst = EFindSqlStateType.stsql;
                      gcurrentsqlstatement.addtokentolist(ast);
                  }

              }


              if (!TBaseType.assigned(gcurrentsqlstatement) ) //error tokentext found
              {

//                  errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                  errorcount = 1;
                  this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                          ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                  ast.tokentype = ETokenType.tttokenlizererrortoken;
                  gst = EFindSqlStateType.sterror;

                  gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                  gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                  gcurrentsqlstatement.addtokentolist(ast);

              }
                break;
            }
            case stsqlplus:{
                if (ast.tokencode  == TBaseType.lexnewline){
                    gst = EFindSqlStateType.stnormal;
                    gcurrentsqlstatement.addtokentolist(ast); // so add it here
                    doongetrawsqlstatementevent(gcurrentsqlstatement);
                }else {
                    {gcurrentsqlstatement.addtokentolist(ast);}
                }

                break;
            }//case source command or \. command
          case stsql :
          {
              if ((ast.tokentype == ETokenType.ttsemicolon)&&(gcurrentsqlstatement.sqlstatementtype != ESqlStatementType.sstmysqldelimiter))
              {
                  gst = EFindSqlStateType.stnormal;
                  gcurrentsqlstatement.addtokentolist(ast);
                  gcurrentsqlstatement.semicolonended = ast;
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  continue;
              }
              if (ast.toString().equalsIgnoreCase(userDelimiterStr))
              {
                  gst = EFindSqlStateType.stnormal;
                  ast.tokencode = ';';// treat it as semicolon
                  gcurrentsqlstatement.addtokentolist(ast);
                  gcurrentsqlstatement.semicolonended = ast;
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  continue;
              }

              if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                  if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                      gst = EFindSqlStateType.stnormal;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                      continue;
                  }
              }

              gcurrentsqlstatement.addtokentolist(ast);

              if ((ast.tokencode  == TBaseType.lexnewline)
                      &&(gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmysqldelimiter)){
                  gst = EFindSqlStateType.stnormal;
                  userDelimiterStr = "";
                  for(int k=0;k<gcurrentsqlstatement.sourcetokenlist.size();k++){
                      TSourceToken st = gcurrentsqlstatement.sourcetokenlist.get(k);
                      if ((st.tokencode == TBaseType.rrw_mysql_delimiter)
                          ||(st.tokencode == TBaseType.lexnewline)
                          ||(st.tokencode == TBaseType.lexspace)
                          ||(st.tokencode == TBaseType.rrw_set) // set delimiter //
                      )
                      {
                          continue;
                      }

                      userDelimiterStr += st.toString();
                      // System.out.println("userDelimiterStr:"+userDelimiterStr);
                     // System.out.print(gcurrentsqlstatement.sourcetokenlist.get(k).toString());
                  }
                  doongetrawsqlstatementevent(gcurrentsqlstatement);

                  continue;
              }

              break;
          }
          case ststoredprocedure :
          {

              if (( gst == EFindSqlStateType.ststoredprocedure ) && (ast.tokencode  == TBaseType.cmtdoublehyphen)){
                  if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                      gst = EFindSqlStateType.stnormal;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                      continue;
                  }
              }
                  // single stmt in function/procedure/trigger may use ; as terminate char
                  // so default terminate char is ;, if begin is found
                  // then set terminate char to DelimiterChar
//                    if (!userDelimiterStr.equals(defaultDelimiterStr) )
//                      if (ast.tokencode == TBaseType.rrw_begin)
//                        userDelimiterStr = defaultDelimiterStr;
              if (waitingDelimiter){
                  if (userDelimiterStr.equalsIgnoreCase(ast.toString())){
                      gst = EFindSqlStateType.stnormal;
                      gcurrentsqlstatement.semicolonended = ast;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                      continue;
                  }else if (userDelimiterStr.startsWith(ast.toString())){
                      String lcstr = ast.toString();
                      for (int k=ast.posinlist+1;k<ast.container.size();k++) {
                          TSourceToken st = ast.container.get(k);
                          if ((st.tokencode == TBaseType.rrw_mysql_delimiter) || (st.tokencode == TBaseType.lexnewline) || (st.tokencode == TBaseType.lexspace)) {
                              break;
                          }
                          lcstr = lcstr + st.toString();
                      }

                      if (userDelimiterStr.equalsIgnoreCase(lcstr)){
                          for (int k=ast.posinlist;k<ast.container.size();k++) {
                              TSourceToken st = ast.container.get(k);
                              if ((st.tokencode == TBaseType.rrw_mysql_delimiter) || (st.tokencode == TBaseType.lexnewline) || (st.tokencode == TBaseType.lexspace)) {
                                  break;
                              }
                              ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                          }
                          gst = EFindSqlStateType.stnormal;
                          gcurrentsqlstatement.semicolonended = ast;
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          continue;
                      }

                  }
              }
                      if (ast.tokencode == TBaseType.rrw_begin)
                          waitingDelimiter = true;

                    if (userDelimiterStr.equals(";")||(waitingDelimiter == false))
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if (ast.tokentype == ETokenType.ttsemicolon)
                        {
                            gst = EFindSqlStateType.stnormal;
                            gcurrentsqlstatement.semicolonended = ast;
                            // asqlstatement._semicolon := ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }
                    else
                    {
                          if (ast.toString().equals(userDelimiterStr) )
                          {
                              ast.tokenstatus = ETokenStatus.tsignorebyyacc;
                              gcurrentsqlstatement.addtokentolist(ast);
                              gst = EFindSqlStateType.stnormal;
                              doongetrawsqlstatementevent(gcurrentsqlstatement);
                          }
                          else{
                              if ((ast.tokentype == ETokenType.ttsemicolon)&&(userDelimiterStr.equals(";")))
                              {
                                  TSourceToken lcprevtoken = ast.container.nextsolidtoken(ast,-1,false);
                                  if (lcprevtoken != null)
                                  {
                                      if (lcprevtoken.tokencode == TBaseType.rrw_end)
                                      {
                                          gst = EFindSqlStateType.stnormal;
                                          gcurrentsqlstatement.semicolonended = ast;
                                          gcurrentsqlstatement.addtokentolist(ast);
                                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                                          continue;
                                      }
                                  }
                              }

                            gcurrentsqlstatement.addtokentolist(ast);
                          }
                    } // end of DelimiterChar
          break;
          } //mysql sp
    } //case
} //for


    //last statement
    if (TBaseType.assigned(gcurrentsqlstatement) && ( (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) || (gst == EFindSqlStateType.sterror))
            ) 
    {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
    }

    return errorcount;
}

int dodb2getrawsqlstatements(){
    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    int i,c;
    TSourceToken ast;
    int errorcount = 0;
    char ch;
    String lcstr;
    int waitingEnds = 0;
    boolean waitingForFirstBegin = false;

    for (i=0 ; i<sourcetokenlist.size();i++)
    {
        ast = sourcetokenlist.get(i);
        //System.out.println(ast);
        sourcetokenlist.curpos = i;

        if (ast.tokencode == TBaseType.rrw_declare){
            TSourceToken st1 = ast.searchToken("global",1);
            if (st1 != null){
              ast.tokencode = TBaseType.rrw_declare_global;
            }
        }else if ((ast.tokencode == TBaseType.rrw_year)||(ast.tokencode == TBaseType.rrw_db2_second)
                ||(ast.tokencode == TBaseType.rrw_db2_current)){
            TSourceToken st1 = ast.searchToken('.',1);
            if (st1 != null){
                // year.columnA, year is a table alias
                ast.tokencode = TBaseType.ident;
            }
        }else if ((ast.tokencode == TBaseType.rrw_db2_trim_l)||(ast.tokencode == TBaseType.rrw_db2_trim_r)){
            TSourceToken st1 = ast.searchToken(TBaseType.rrw_db2_trim,-2);
            if (st1 == null){
                // not L,R in TRIM (R 'Mr.' FROM name)
                ast.tokencode = TBaseType.ident;
            }
        }

        switch(gst){
          case sterror:
          {
              if (ast.tokentype == ETokenType.ttsemicolon)
              {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
              }
              else
              {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
              }
              break;
          }
          case stnormal :
          {
              if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                 || (ast.tokencode  == TBaseType.cmtslashstar)
                 || (ast.tokencode  == TBaseType.lexspace)
                 || (ast.tokencode  == TBaseType.lexnewline)
                 || (ast.tokentype  == ETokenType.ttsemicolon))
                 {
                 if (TBaseType.assigned(gcurrentsqlstatement))
                     {
                     gcurrentsqlstatement.addtokentolist(ast);
                     }
                 continue;
                 }

              if (ast.tokencode == TBaseType.sqlpluscmd )
              { // echo in db2 is treated as sqlplus cmd
                  gst = EFindSqlStateType.stsqlplus;
                  gcurrentsqlstatement = new TSqlplusCmdStatement(dbVendor);
                  gcurrentsqlstatement.addtokentolist(ast);
                  continue;
              }

              // find a tokentext to start sql or plsql mode
              gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

              if (TBaseType.assigned(gcurrentsqlstatement))
              {
                  ESqlStatementType[] ses = {ESqlStatementType.sstdb2declarecursor, sstcreateprocedure,
                          ESqlStatementType.sstcreatefunction, ESqlStatementType.sstcreatetrigger,ESqlStatementType.sst_plsql_block};
                  if(includesqlstatementtype(gcurrentsqlstatement.sqlstatementtype, ses) )
                       {
                        gst = EFindSqlStateType.ststoredprocedure;
                        waitingEnds = 1; //need a END keyword to end this procedure
                        waitingForFirstBegin = true;
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((gcurrentsqlstatement.sqlstatementtype == sstcreateprocedure)
                        || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatefunction)
                        || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatetrigger))
                           {
                            curdelimiterchar = ';';
                           }
                        }
                  else if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstdb2scriptOption){
                      gst = EFindSqlStateType.stnormal;
                      gcurrentsqlstatement.addtokentolist(ast);
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                  }
                  else  {
                      gst = EFindSqlStateType.stsql;
                      gcurrentsqlstatement.addtokentolist(ast);
                    }

              }


              if (!TBaseType.assigned(gcurrentsqlstatement) ) //error tokentext found
              {

//                  errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                  errorcount = 1;
                  this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                          ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                  ast.tokentype = ETokenType.tttokenlizererrortoken;
                  gst = EFindSqlStateType.sterror;

                  gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                  gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                  gcurrentsqlstatement.addtokentolist(ast);

              }
              break;
          }
          case stsqlplus : //// treat echo, ! of db2 at the begin of each line as oracle sqlplus command
          {
              if (ast.insqlpluscmd)
              { gcurrentsqlstatement.addtokentolist(ast);}
              else
              {
                  gst = EFindSqlStateType.stnormal; //this tokentext must be newline,
                  gcurrentsqlstatement.addtokentolist(ast); // so add it here
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
              }
              break;
          }
          case stsql :
          {
              if (ast.tokentype == ETokenType.ttsemicolon)
              {
                  gst = EFindSqlStateType.stnormal;
                  gcurrentsqlstatement.addtokentolist(ast);
                  gcurrentsqlstatement.semicolonended = ast;
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  continue;
              }
              gcurrentsqlstatement.addtokentolist(ast);

              break;
          }
          case ststoredprocedure :
          {
              nextStmt = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);
              if ((nextStmt != null)
                      &&((nextStmt.sqlstatementtype == sstcreateprocedure)
                      || (nextStmt.sqlstatementtype == ESqlStatementType.sstcreatefunction)
                      || (nextStmt.sqlstatementtype == ESqlStatementType.sstcreatetrigger))
                      )
              {
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gcurrentsqlstatement = nextStmt;
                  gst = EFindSqlStateType.ststoredprocedure;
                  waitingEnds = 1; //need a END keyword to end this procedure
                  waitingForFirstBegin = true;
                  gcurrentsqlstatement.addtokentolist(ast);
                  curdelimiterchar = ';';
                  continue;
              }

                  // single stmt in function/procedure/trigger may use ; as terminate char
                  // so default terminate char is ;, if begin is found ,then
                  //set terminate char to DelimiterChar
                    if (curdelimiterchar !=  delimiterchar)
//                    if TBaseType.mycomparetext(lzWideLowerCase(ast.astext),'begin') = 0 then
                      if((ast.tokencode == TBaseType.rrw_begin )||(ast.tokencode == TBaseType.rrw_declare ))
                      {
                          curdelimiterchar = delimiterchar;
                      }


                    if (curdelimiterchar == ';')
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if (ast.tokentype == ETokenType.ttsemicolon)
                        {
                            gst = EFindSqlStateType.stnormal;
                            //asqlstatement.SemiColonEnded := ast;
                             gcurrentsqlstatement._semicolon = ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }
                    else
                    {
                          if (ast.astext.length() == 1 )
                          { ch = ast.astext.charAt(0);}
                          else if ((ast.astext.length() > 1)&&(ast.issolidtoken()))
                           { ch = ast.astext.charAt(ast.astext.length()-1); }
                          else
                          { ch = ' ';}


                          if (ch == curdelimiterchar )
                          {
                              if (ast.astext.length() > 1)
                              {
                                  lcstr = ast.astext.substring(0,ast.astext.length()-1);
                                  if ((c=flexer.getkeywordvalue(lcstr))>0 )
                                    ast.tokencode = c;
                              }
                              else
                              {
                                 // ast.TokenStatus := tsIgnoreByyacc;
                                 // ast.Tokentype :=   ttDoublehyphenComment; //make this tokentext available like a comment
                                  gcurrentsqlstatement._semicolon = ast;
                              }
                              gcurrentsqlstatement.addtokentolist(ast);
                              gst = EFindSqlStateType.stnormal;
                              doongetrawsqlstatementevent(gcurrentsqlstatement);
                          }
                          else
                          {
                              gcurrentsqlstatement.addtokentolist(ast);

                              if((ast.tokencode == TBaseType.rrw_case)
                                      ||(ast.tokencode == TBaseType.rrw_for)
                                      ||(ast.tokencode == TBaseType.rrw_if)
                                      ||(ast.tokencode == TBaseType.rrw_while)
                                      ||(ast.tokencode == TBaseType.rrw_repeat)
                                      ||(ast.tokencode == TBaseType.rrw_loop)
                              ){
                                  TSourceToken nextst = ast.nextSolidToken();
                                  if (nextst != null){
                                      if (nextst.tokencode != ';'){
                                          waitingEnds++;
                                      }
                                  }

                              } else if(ast.tokencode == TBaseType.rrw_begin){
                                  if (waitingForFirstBegin){
                                      waitingForFirstBegin = false;
                                  }else{
                                      waitingEnds++;
                                  }
                              } else if(ast.tokencode == TBaseType.rrw_end){
                                  waitingEnds--;
                              }

                              if ((ast.tokentype == ETokenType.ttsemicolon)&&(waitingEnds == 0))
                              {
                                  gst = EFindSqlStateType.stnormal;
                                  //asqlstatement.SemiColonEnded := ast;
                                   gcurrentsqlstatement._semicolon = ast;
                                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                                  continue;
                              }

                          }
                    } // end of DelimiterChar

              break;
          } //db2 sp
        } //case
    } //for


    //last statement
    if (TBaseType.assigned(gcurrentsqlstatement) && ((gst == EFindSqlStateType.stsqlplus) ||  (gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) || (gst == EFindSqlStateType.sterror)) )
    {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
    }

    return errorcount;//FSqlStatements.Count;

}

int doteradatagetrawsqlstatements(){
    gcurrentsqlstatement = null;
    EFindSqlStateType gst = EFindSqlStateType.stnormal;
    int i,c,lcNestedParens = 0, lcNestedBeginEnd = 0, lcNestedCase = 0, lcNestedSample = 0;
    TSourceToken lcprevst = null,lcnextst=null,lcnextst2= null, lcnextst3 = null,
            lcprevsolidtoken = null,ast = null,lcprevst2 = null;
    int errorcount = 0;
    char ch;
    String lcstr;
    boolean inBTEQComment = false, isContinueBTEQCmd = false;

    for (i=0 ; i<sourcetokenlist.size();i++)
    {
        if ( (ast != null ) && (ast.issolidtoken() ))
          lcprevsolidtoken = ast;

        ast = sourcetokenlist.get(i);
        //System.out.println(ast);
        sourcetokenlist.curpos = i;

        if ((ast.tokencode == TBaseType.rrw_time)
                ||(ast.tokencode == TBaseType.rrw_date)
                ||(ast.tokencode == TBaseType.rrw_timestamp)
           )
            {
            }else if (ast.tokencode == '('){
                    lcnextst = sourcetokenlist.nextsolidtoken(i,1,false);
                    lcprevst =  getprevsolidtoken(ast);
                    if ((lcprevst != null)&&(
                                (lcprevst.tokencode == TBaseType.rrw_teradata_cast)
                                    ||(lcprevst.tokencode == TBaseType.rrw_year) // sample sql: select year(date) - 1 mantisbt/view.php?id=3049
                            )
                        ){
                        lcnextst = null; // don't check next token if prev token is cast. sample: cast(DATETIME as BIGINT)
                    }
                    if ((lcprevst!=null)&&(TFunctionCall.isBuiltIn(lcprevst.toString(),EDbVendor.dbvteradata))){
                        if (((lcprevst.tokencode == TBaseType.rrw_end)
                                ||(lcprevst.tokencode == TBaseType.rrw_time)
                                ||(lcprevst.tokencode == TBaseType.rrw_date)
                                ||(lcprevst.tokencode == TBaseType.rrw_timestamp)
                                ||(lcprevst.tokencode == TBaseType.rrw_teradata_type)
                                ||(lcprevst.toString().equalsIgnoreCase("current_date"))
                                ||(lcprevst.toString().equalsIgnoreCase("current_time"))
                                ||(lcprevst.toString().equalsIgnoreCase("current_timestamp"))
                                ||(lcprevst.toString().equalsIgnoreCase("user"))
                                ||(lcprevst.toString().equalsIgnoreCase("current_user"))
                                ))
                        {

                        }else{
                            lcnextst = null;
                        }
                    }

                    if (lcnextst != null){
                        EDataType dataType = TTypeName.searchTeradataTypeByName(lcnextst.toString());
                        if (( dataType != null)&&(lcprevst!=null)&&(lcprevst.tokencode != TBaseType.rrw_teradata_period)){
                            ast.tokencode = TBaseType.rrw_teradata_start_data_conversion;
                            // lcnextst2 is the token 2 steps after (
                            lcnextst2 = sourcetokenlist.nextsolidtoken(i,2,false);

                            // lcnextst3 is the token 3 steps after (
                            lcnextst3 = sourcetokenlist.nextsolidtoken(i,3,false);

                            if (lcnextst2.tokencode == ',') {
                                // this is data conversion: (date,format '$xxx'),
                                // this is not data conversion: trunc(date,'yy')
                                TSourceToken st1 = lcnextst3;
                                TSourceToken st2 = sourcetokenlist.nextsolidtoken(i, 4, false);
                                if (TDatatypeAttribute.searchDataTypeAttributeByName(st1, st2) != null) {
                                    // this is data conversion, such as (date,format '$xxx'),
                                } else {
                                    // this is not data conversion, such as :trunc(date,'yy')
                                    ast.tokencode = '(';
                                }
                            }else if ((lcnextst2 != null)&&(lcnextst3 != null)){
                                switch (dataType){
                                    case date_t:
                                    case time_t:
                                        // next token is: ) or comma or data attribute token
                                        if ((lcnextst2.tokencode == ')')
                                                    ||((lcnextst2.tokencode == '(')&&(lcnextst3.tokencode == 263)) // hire_date (time(6))
                                        ) {
                                            //  (date)
                                            // select (date) from t, here (date) is a function, not a data conversion,
                                            //

                                        }else {
                                            ast.tokencode = '(';
                                        }
                                        break;
                                    case timestamp_t:
                                    case interval_t:
                                        if (lcnextst2.tokencode == TBaseType.sconst){ // INSERT t1 (TIME '10:44:25.123-08:00',TIMESTAMP '2000-09-20 10:44:25.1234')
                                            ast.tokencode = '(';
                                        }
                                            break;
                                    case period_t:

                                        if (lcnextst2.tokencode == '('){
                                            // 只有 period() 才确保这里 period 是datatype，否则period不是datatype
                                        }else{
                                            ast.tokencode = '(';
                                        }
                                        break;
                                    case char_t:
                                        if (lcnextst2.tokencode == ')'){
                                            //  NULL (CHAR) AS B

                                        }
                                        else if (lcnextst3.tokencode != TBaseType.iconst){ // INSERT t1 (TIME '10:44:25.123-08:00',TIMESTAMP '2000-09-20 10:44:25.1234')
                                            ast.tokencode = '(';
                                        }
                                        break;
                                    default:
                                        break;
                                }
                            }
                        }else{
//                            lcstr = lcnextst.toString();
//                            if ((lcnextst.tokencode == TBaseType.rrw_not)||(lcnextst.tokencode == TBaseType.rw_not1) // not null, not casespecific
//                                ||(lcnextst.tokencode == TBaseType.rrw_with) // with default, with time zone
//                            ){
//                                // data attribute include more than one keyword, concate the first 2 keywords
//                                lcnextst = sourcetokenlist.nextsolidtoken(i, 2, false);
//                                if (lcnextst != null){
//                                    lcstr = lcstr+"_"+lcnextst.toString();
//                                }else{
//                                    lcstr = lcstr+"_";
//                                }
//                            }

                            TSourceToken st1 = lcnextst;
                            TSourceToken st2 = sourcetokenlist.nextsolidtoken(i, 2, false);
                            if (TDatatypeAttribute.searchDataTypeAttributeByName(st1,st2) != null){
                                ast.tokencode = TBaseType.rrw_teradata_start_data_conversion;
                            }
                        }
                    }
        }else if (ast.tokencode == TBaseType.rrw_for){
            TSourceToken nextToken = ast.searchTokenAfterObjectName();
            if (nextToken != null){
                if (nextToken.tokencode == TBaseType.rrw_as){
                    ast.tokencode = TBaseType.rrw_teradata_for_loop;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_teradata_last){
            TSourceToken nextToken = ast.searchToken('(',1);
            if (nextToken != null){
               ast.tokencode = TBaseType.rrw_last_function;
            }
        }else if (ast.tokencode == TBaseType.rrw_teradata_pivot){
            TSourceToken nextToken = ast.searchToken('(',1);
            if (nextToken == null){
                ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_teradata_period){
            TSourceToken nextToken =  ast.nextSolidToken(); // ast.searchToken('(',1);
            if (nextToken == null){
                ast.tokencode = TBaseType.ident;
            }else if (nextToken.tokencode == '('){
                // keep it as keyword
            }else if (nextToken.tokencode == TBaseType.rrw_for){
                // keep it as keyword
            }else if (nextToken.tokencode == TBaseType.sconst){
                // sample: DEFAULT PERIOD '(2005-02-03, 2006-02-03)'
                // keep it as keyword
            }else {
                if (ast.prevSolidToken() == null){
                    ast.tokencode = TBaseType.ident;
                }else{
                    if (!ast.prevSolidToken().toString().equalsIgnoreCase("anchor")){
                        ast.tokencode = TBaseType.ident;
                    }
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_teradata_transaction) {
            lcprevst =  getprevsolidtoken(ast);
            if (lcprevst != null) {
                if (lcprevst.tokencode == TBaseType.rrw_end){
                    lcprevst.tokencode = TBaseType.rrw_teradata_end_t;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_replace) {
            TSourceToken nextToken = ast.searchToken('(',1);
            if (nextToken != null){
               ast.tokencode = TBaseType.ident;
            }
        }else if (ast.tokencode == TBaseType.rrw_between) {
            TSourceToken nextToken = ast.nextSolidToken();
            if (nextToken != null){
                if ((nextToken.tokencode == TBaseType.rrw_begin)||(nextToken.tokencode == TBaseType.rrw_end)){
                    // BETWEEN begin(DODD_EFF_PERI)
                    nextToken.tokencode = TBaseType.ident;
                }
            }
        }else if (ast.tokencode == TBaseType.rrw_declare) {
            TSourceToken nextToken = ast.nextSolidToken();
            if (nextToken != null){
                TSourceToken nextnextToken = nextToken.nextSolidToken();
                if (nextnextToken != null){
                    if ((nextnextToken.toString().equalsIgnoreCase("cursor"))
                    ||(nextnextToken.toString().equalsIgnoreCase("insensitive"))
                            ||(nextnextToken.toString().equalsIgnoreCase("scroll"))
                            ||(nextnextToken.toString().equalsIgnoreCase("no"))
                    ){
                        nextToken.tokencode = TBaseType.rrw_teradata_cursor_name;
                    }else if ((nextnextToken.toString().equalsIgnoreCase("condition"))){
                        nextToken.tokencode = TBaseType.rrw_teradata_condition_name;
                    }
                }
            }
        }else if ((ast.tokencode == TBaseType.rrw_case)&&(ast.tag == 0)&&(ast.posinlist + 1 < sourcetokenlist.size())) {
            int lcNest = 1;
            for(int m=ast.posinlist+1;m< sourcetokenlist.size()-1;m++){
                TSourceToken st = sourcetokenlist.get(m);
                if (st.tokencode == TBaseType.rrw_case) {
                    st.tag = 1;// this is nested case keyword, don't check this token later
                    lcNest++;
                }
                if (st.tokencode == TBaseType.rrw_end) {
                    lcNest--;
                    if (lcNest == 0){
                        TSourceToken caseToken = st.nextSolidToken();
                        if (caseToken != null){
                            if (caseToken.tokencode == TBaseType.rrw_case){
                                ast.tokencode = TBaseType.rrw_teradata_case_stmt;
                                caseToken.tag = 1; // this is case after end, don't check this token later
                                break;
                            }
                        }
                    }
                }
            }
        } else if (ast.tokencode == TBaseType.rrw_grant) {
            TSourceToken prevSolidToken = ast.prevSolidToken();
            if (prevSolidToken != null) {
                if (prevSolidToken.tokencode == TBaseType.rrw_with){
                    prevSolidToken.tokencode = TBaseType.rrw_teradata_with_grant;
                }
            }
        } else if (ast.tokencode == TBaseType.variable) {
            // USING ( _spVV8 VARCHAR(1024) CHARACTER SET UNICODE ) SELECT INTO :_spVV8 ;
            // variable after into keyword treat as identifier

            TSourceToken prevSolidToken = ast.prevSolidToken();
            if (prevSolidToken != null) {
                if (prevSolidToken.tokencode == TBaseType.rrw_into){
                    ast.tokencode = TBaseType.ident;
                }
            }
        }else if ((ast.tokencode == TBaseType.rrw_teradata_sequenced)||(ast.tokencode == TBaseType.rrw_teradata_nonsequenced)) {
            TSourceToken nextSolidToken = ast.nextSolidToken();
            if (nextSolidToken != null) {
                if ((nextSolidToken.tokencode == TBaseType.rrw_teradata_validtime)){

                    TSourceToken nextToken2 = nextSolidToken.nextSolidToken();
                    if (nextToken2 != null) {
                        if ((nextToken2.tokencode == TBaseType.rrw_select)
                                ||(nextToken2.tokencode == TBaseType.rrw_insert)
                                ||(nextToken2.tokencode == TBaseType.rrw_update)
                                ||(nextToken2.tokencode == TBaseType.rrw_delete)
                                ||(nextToken2.tokencode == TBaseType.rrw_merge)){
                            // SEQUENCED VALIDTIME select/insert/update/delete/merge
                            ast.tokencode = TBaseType.lexspace;
                            nextSolidToken.tokencode = TBaseType.lexspace;
                        }
                    }
                }
            }
        }


        switch(gst){
          case sterror:
          {
              if (ast.tokentype == ETokenType.ttsemicolon)
              {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
              }
              else
              {
                  gcurrentsqlstatement.sourcetokenlist.add(ast);
              }
              break;
          }
          case stMultiLoadCmd: {
              gcurrentsqlstatement.addtokentolist(ast);
             if ((ast.tokencode == ';')&&(ast.isLastTokenOfLine())) {
                 TSourceToken nextToken = ast.nextSolidToken();
                 TSourceToken nextToken2 = nextToken != null ? nextToken.nextSolidToken() : null;
                 if (nextToken != null && nextToken2 != null && 
                     nextToken.tokencode == '.' && 
                     nextToken2.toString().equalsIgnoreCase("FIELD") &&
                     nextToken.isFirstTokenOfLine()) {
                     // Remain in stMultiLoadCmd state
                 } else {
                     doongetrawsqlstatementevent(gcurrentsqlstatement);
                     gst = EFindSqlStateType.stnormal;
                 }
             }else if (ast.isLastTokenOfLine()){
                 //.IF ERRORCODE <> 0 THEN
                 //INSERT INTO Sou_EMP_Tab
                   //      (1, 'bala');
                 String cmdstr = gcurrentsqlstatement.sourcetokenlist.get(0).toString();
                 if (gcurrentsqlstatement.sourcetokenlist.size() > 1){
                     cmdstr = cmdstr+gcurrentsqlstatement.sourcetokenlist.get(1).toString();
                 }
                 if ((cmdstr.toLowerCase().startsWith(".if"))||(cmdstr.toLowerCase().startsWith(".set"))||(cmdstr.toLowerCase().startsWith("set"))){
                     doongetrawsqlstatementevent(gcurrentsqlstatement);
                     gst = EFindSqlStateType.stnormal;
                 }
             }
              //System.out.println("find stMultiLoadCmd");
              break;
          }
          case stFastExportCmd: {
              gcurrentsqlstatement.addtokentolist(ast);
              if ((ast.tokencode == ';')&&(ast.isLastTokenOfLine())) {
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
              }else{
                  boolean findSQL = false;
                  TSourceToken nextst = ast.nextToken();
                  if (nextst != null){
                      if (nextst.tokentype == ETokenType.ttreturn){
                          findSQL = (sqlcmds.issql(ast.nextSolidToken(), dbVendor,gst,gcurrentsqlstatement) != null);
                          if (findSQL){
                              doongetrawsqlstatementevent(gcurrentsqlstatement);
                              gst = EFindSqlStateType.stnormal;
                          }
                      }
                  }
              }
              //System.out.println("find stFastExportCmd");
              break;
          }
          case stFastLoadCmd: {
              gcurrentsqlstatement.addtokentolist(ast);
              if ((ast.tokencode == ';')&&(ast.isLastTokenOfLine())) {
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  gst = EFindSqlStateType.stnormal;
              }else if (ast.tokencode == TBaseType.lexnewline){
                  TSourceToken nextst = ast.nextSolidToken();
                  if (nextst != null){
                      if (nextst.tokencode == TBaseType.BTEQCMD){
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gst = EFindSqlStateType.stFastLoadCmd;
                      }else if (nextst.tokencode == '.'){
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gst = EFindSqlStateType.stFastLoadCmd;
                      }else if (sqlcmds.issql(nextst, dbVendor,gst,gcurrentsqlstatement) != null){
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gst = EFindSqlStateType.stnormal;
                      }
                  }
              }
              //System.out.println("find stFastLoadCmd");
              break;
          }
          case stBTEQCmd:
          {
              gcurrentsqlstatement.addtokentolist(ast);

              if (ast.tokencode == TBaseType.lexnewline){
                  if (isContinueBTEQCmd){
                      isContinueBTEQCmd = false;
                  }else{
                     // System.out.println(((TTeradataBTEQCmd)gcurrentsqlstatement).getBteqCmdType());
                      if (((TTeradataBTEQCmd)gcurrentsqlstatement).getBteqCmdType() == BteqCmdType.LOGON){
                          //TSourceToken nextSolidToken = ast.nextSolidToken()
                          boolean findSQL = false;
                           findSQL = (sqlcmds.issql(ast.nextSolidToken(), dbVendor,gst,gcurrentsqlstatement) != null);
                          if (findSQL){
                              doongetrawsqlstatementevent(gcurrentsqlstatement);
                              gst = EFindSqlStateType.stnormal;
                          }
                      }else{
                          doongetrawsqlstatementevent(gcurrentsqlstatement);
                          gst = EFindSqlStateType.stnormal;
                      }
                  }
              }else if ((ast.tokencode == '-')&&(ast.isLastTokenOfLine())){
                  if (!inBTEQComment)
                    isContinueBTEQCmd = true;
              }else if (ast.tokencode  == TBaseType.lexspace){

              }else if (ast.tokencode  == ';'){

                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  boolean findNewCmdInSameLine = false;
                  if (!ast.isLastTokenOfLine()){
                      // check if next cmd like below:
                      // .SET SIDETITLES ON; .SET FOLDLINE ON ALL
                      TSourceToken nextst  = ast.nextSolidToken();
                      if ((nextst != null)&&(nextst.tokencode == '.')){
                          TSourceToken nextst2  = nextst.nextSolidToken();
                          TSourceToken searchToken = null;
                          if (nextst2 != null){

                              String searchStr = null;
                              if (nextst2.tokencode == TBaseType.rrw_set){
                                  TSourceToken nextst3 = nextst2.nextSolidToken();
                                  if (nextst3 != null){
                                      searchStr = nextst3.astext;
                                      searchToken = nextst3;
                                  }

                              }else{
                                  searchStr = nextst2.astext;
                                    searchToken = nextst2;
                              }

                              //EBTEQCmdType ebteqCmdType = TTeradataHelper.searchBTEQTypeByName(searchStr);
                              BteqCmdType bteqCmdType = BteqCmdType.searchBteqCmd(searchToken);
                              if (bteqCmdType != null){
                                  gcurrentsqlstatement = new TTeradataBTEQCmd(dbVendor);
                                  ((TTeradataBTEQCmd)gcurrentsqlstatement).setBteqCmdType(bteqCmdType);
                                  gcurrentsqlstatement.addtokentolist(ast);
                                  findNewCmdInSameLine = true;
                              }
                          }
                      }
                  }

                  if (findNewCmdInSameLine){
                      gst = EFindSqlStateType.stBTEQCmd;
                  }else{
                      gst = EFindSqlStateType.stnormal;
                  }

              }else if (ast.tokencode  == '*'){
//                  if (inBTEQComment){
//                      inBTEQComment = false;
//                      doongetrawsqlstatementevent(gcurrentsqlstatement);
//                      gst = EFindSqlStateType.stnormal;
//                  }
                  isContinueBTEQCmd = false;
              }else{ //solid token
                  isContinueBTEQCmd = false;
              }
              //System.out.println("find stBTEQCmd");
              break;
          }
          case stnormal :
          {
              if ((ast.tokencode  == TBaseType.cmtdoublehyphen)
                 || (ast.tokencode  == TBaseType.cmtslashstar)
                 || (ast.tokencode  == TBaseType.lexspace)
                 || (ast.tokencode  == TBaseType.lexnewline)
                 || (ast.tokentype  == ETokenType.ttsemicolon))
                 {
                 if (TBaseType.assigned(gcurrentsqlstatement))
                     {
                     gcurrentsqlstatement.addtokentolist(ast);
                     }

                     if ((lcprevsolidtoken != null) && (ast.tokentype == ETokenType.ttsemicolon))
                     {
                         if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon )
                         {
                            // ;;;; continuous semicolon,treat it as comment
                             ast.tokentype = ETokenType.ttsimplecomment;
                             ast.tokencode =  TBaseType.cmtdoublehyphen;
                         }
                     }

                 continue;
                 }


              if ((ast.tokencode == '.')&&(ast.isFirstTokenOfLine())){
                  TSourceToken nextst  = ast.nextSolidToken();
                  TSourceToken cmdToken = null;
                  if (nextst != null){

                    //   String searchStr = null;
                    //   if (nextst.tokencode == TBaseType.rrw_set){
                    //       TSourceToken nextst2 = nextst.nextSolidToken();
                    //       if (nextst2 != null){
                    //           searchStr = nextst2.astext;
                    //           cmdToken = nextst2;
                    //       }

                    //   }else{
                    //       searchStr = nextst.astext;
                    //       cmdToken = nextst;
                    //   }

                    //   EBTEQCmdType ebteqCmdType = TTeradataHelper.searchBTEQTypeByName(searchStr);
                    //   if (ebteqCmdType != null){
                    //       //check is this a bteq command
                    //       //if (ebteqCmdType == EBTEQCmdType.EXPORT)
                    //       TSourceToken nextst2;
                    //       switch (ebteqCmdType){
                    //           case EXPORT:
                    //               nextst2 = cmdToken.nextSolidToken();
                    //               if (nextst2 != null){
                    //                   if ((nextst2.astext.equalsIgnoreCase("data"))
                    //                       ||(nextst2.astext.equalsIgnoreCase("indicdata"))
                    //                           ||(nextst2.astext.equalsIgnoreCase("report"))
                    //                           ||(nextst2.astext.equalsIgnoreCase("dif"))
                    //                           ||(nextst2.astext.equalsIgnoreCase("reset"))
                    //                   ){

                    //                   }else{
                    //                       ebteqCmdType = null;
                    //                   }

                    //               }
                    //             break;
                    //           case IMPORT:
                    //               nextst2 = cmdToken.nextSolidToken();
                    //               if (nextst2 != null){
                    //                   if ((nextst2.astext.equalsIgnoreCase("data"))
                    //                           ||(nextst2.astext.equalsIgnoreCase("indicdata"))
                    //                           ||(nextst2.astext.equalsIgnoreCase("vartext"))
                    //                           ||(nextst2.astext.equalsIgnoreCase("reset"))
                    //                           ||(nextst2.astext.equalsIgnoreCase("ddname"))
                    //                   ){

                    //                   }else{
                    //                       ebteqCmdType = null;
                    //                   }
                    //               }
                    //               break;
                    //           case SET_SESSION:
                    //               nextst2 = cmdToken.nextSolidToken();
                    //               if (nextst2 != null){
                    //                   if (nextst2.astext.equalsIgnoreCase("charset")) {
                    //                       ebteqCmdType = EBTEQCmdType.SESSION_CHARSET;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("RESPBUFLEN")){
                    //                       ebteqCmdType = EBTEQCmdType.SESSION_RESPBUFLEN;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("SQLFLAG")){
                    //                       ebteqCmdType = EBTEQCmdType.SESSION_SQLFLAG;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("TRANSACTION")){
                    //                       ebteqCmdType = EBTEQCmdType.SESSION_TRANSACTION;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("TRANS")){
                    //                       ebteqCmdType = EBTEQCmdType.SESSION_TRANSACTION;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("TWORESPBUFS")){
                    //                       ebteqCmdType = EBTEQCmdType.SESSION_TWORESPBUFS;
                    //                   }else{
                    //                       ebteqCmdType = null;
                    //                   }
                    //               }
                    //               break;
                    //           case SHOW:
                    //               nextst2 = cmdToken.nextSolidToken();
                    //               if (nextst2 != null){
                    //                   if (nextst2.astext.equalsIgnoreCase("CONTROLS")) {
                    //                       ebteqCmdType = EBTEQCmdType.SHOW_CONTROLS;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("CONTROL")){
                    //                       ebteqCmdType = EBTEQCmdType.SHOW_CONTROLS;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("ERRORMAP")){
                    //                       ebteqCmdType = EBTEQCmdType.SHOW_ERRORMAP;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("VERSION")){
                    //                       ebteqCmdType = EBTEQCmdType.SHOW_VERSIONS;
                    //                   }else if (nextst2.astext.equalsIgnoreCase("VERSIONS")){
                    //                       ebteqCmdType = EBTEQCmdType.SHOW_VERSIONS;
                    //                   }
                    //               }
                    //               break;
                    //       }

                    //       if (ebteqCmdType != null){
                    //           gcurrentsqlstatement = new TTeradataBTEQCmd(dbVendor);
                    //           ((TTeradataBTEQCmd)gcurrentsqlstatement).setBteqCmdType(ebteqCmdType);
                    //           gcurrentsqlstatement.addtokentolist(ast);
                    //           gst = EFindSqlStateType.stBTEQCmd;
                    //           continue;
                    //       }
                    //   }

                      gst = ((TLexerTeradata)getFlexer()).cmdType(nextst);
                      boolean foundCmd = false;
                      switch (gst){
                          case stMultiLoadCmd:
                              gcurrentsqlstatement = new TTeradataMultiLoadCmd(dbVendor);
                              gst = EFindSqlStateType.stMultiLoadCmd;
                              ((TTeradataMultiLoadCmd)gcurrentsqlstatement).setCmdType(TTeradataHelper.searchMultiLoadTypeByName(nextst.astext));
                              foundCmd = true;
                              break;
                          case stFastExportCmd:
                              gcurrentsqlstatement = new TTeradataFastExportCmd(dbVendor);
                              gst = EFindSqlStateType.stFastExportCmd;
                              foundCmd = true;
                              break;
                          case stFastLoadCmd:
                              gcurrentsqlstatement = new TTeradataFastLoadCmd(dbVendor);
                              gst = EFindSqlStateType.stFastLoadCmd;
                              foundCmd = true;
                              break;
                          case stBTEQCmd:
                              gcurrentsqlstatement = new TTeradataBTEQCmd(dbVendor);
                              ((TTeradataBTEQCmd)gcurrentsqlstatement).setBteqCmdType(BteqCmdType.searchBteqCmd(nextst));
                              gst = EFindSqlStateType.stBTEQCmd;
                              foundCmd = true;
                              break;
                      }
                      if (foundCmd) {
                          gcurrentsqlstatement.addtokentolist(ast);
                          continue;
                      }

                  } // next solid token
              }
              else if ((ast.tokencode == '*')&&(ast.isFirstTokenOfLine())){
                  gst = EFindSqlStateType.stBTEQCmd;
                  inBTEQComment = true;
                  gcurrentsqlstatement = new TTeradataBTEQCmd(dbVendor);
                  ((TTeradataBTEQCmd)gcurrentsqlstatement).setBteqCmdType(BteqCmdType.COMMENT);
                  //((TTeradataBTEQCmd)gcurrentsqlstatement).setFindSqlStateType(gst);
                  gcurrentsqlstatement.addtokentolist(ast);
                  continue;
              }
              else if ((ast.tokencode == '=')&&(ast.isFirstTokenOfLine())){
                  TSourceToken st1 = ast.prevSolidToken();
                  if (st1 != null){
                      if (st1.tokencode == ';'){
                          gst = EFindSqlStateType.stBTEQCmd;
                          gcurrentsqlstatement = new TTeradataBTEQCmd(dbVendor);
                          ((TTeradataBTEQCmd)gcurrentsqlstatement).setFindSqlStateType(gst);
                          gcurrentsqlstatement.addtokentolist(ast);
                          continue;
                      }
                  }
              }
              else if (ast.isFirstTokenOfLine() && (BteqCmdType.searchBteqCmd(ast) != null)){
                  TSourceToken st1 = ast.prevSolidToken();
                  if (st1 != null){
                      if (st1.tokencode == ';'){
                          gst = EFindSqlStateType.stBTEQCmd;
                          gcurrentsqlstatement = new TTeradataBTEQCmd(dbVendor);
                          ((TTeradataBTEQCmd)gcurrentsqlstatement).setFindSqlStateType(gst);
                          gcurrentsqlstatement.addtokentolist(ast);
                          continue;
                      }
                  }
              }


              // find a tokentext to start sql or stored procedure mode
              gcurrentsqlstatement = sqlcmds.issql(ast, dbVendor,gst,gcurrentsqlstatement);

              if (TBaseType.assigned(gcurrentsqlstatement))
              {
                  ESqlStatementType[] ses = { sstcreateprocedure
                          ,ESqlStatementType.sstteradatacreatefunction,ESqlStatementType.sstcreatetrigger};
                  if(includesqlstatementtype(gcurrentsqlstatement.sqlstatementtype, ses) )
                       {
                        gst = EFindSqlStateType.ststoredprocedure;
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((gcurrentsqlstatement.sqlstatementtype == sstcreateprocedure)
                        || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstteradatacreatefunction)
                        || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatetrigger)
                        )
                           {
                            curdelimiterchar = ';';
                           }
                        }
                  else
                    {
                      gst = EFindSqlStateType.stsql;
                      gcurrentsqlstatement.addtokentolist(ast);
                    }

                  lcNestedParens = 0;

              }

              if ((!TBaseType.assigned(gcurrentsqlstatement) ) && (ast.isFirstTokenOfLine())){
                  gst = ((TLexerTeradata)getFlexer()).cmdType(ast);
                  if ((gst == EFindSqlStateType.stMultiLoadCmd)||(gst == EFindSqlStateType.stFastLoadCmd)||(gst == EFindSqlStateType.stFastExportCmd)){
                      gcurrentsqlstatement = new TTeradataBTEQCmd(dbVendor);
                      ((TTeradataBTEQCmd)gcurrentsqlstatement).setFindSqlStateType(gst);
                      gcurrentsqlstatement.addtokentolist(ast);
                      continue;
                  }else gst = EFindSqlStateType.stnormal;
              }

              if (!TBaseType.assigned(gcurrentsqlstatement) ) //error tokentext found
              {

//                  errormessage = "error when tokenlize:"+ast.astext+"("+ast.lineNo +","+ast.columnNo +")";
//                  errorcount = 1;
                  this.syntaxErrors.add( new TSyntaxError(ast.astext,ast.lineNo,(ast.columnNo < 0 ? 0:ast.columnNo)
                          ,"Error when tokenlize", EErrorType.spwarning,TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE,null,ast.posinlist));

                  ast.tokentype = ETokenType.tttokenlizererrortoken;
                  gst = EFindSqlStateType.sterror;

                  gcurrentsqlstatement = new TUnknownSqlStatement(dbVendor);
                  gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
                  gcurrentsqlstatement.addtokentolist(ast);

              }
              break;
          }
          case stsql :
          {
              boolean readyToEnd = true;
              if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstteradatacreatemacro){
                  if ((ast.tokencode == '(')||(ast.tokencode == TBaseType.left_parenthesis_2)) lcNestedParens++;
                  if (ast.tokencode == ')') {
                      lcNestedParens--;
                      if (lcNestedParens < 0) lcNestedParens = 0;
                  }

                  readyToEnd = (lcNestedParens == 0);
              }

              if ((ast.tokentype == ETokenType.ttsemicolon) && (readyToEnd))
              {
                  gst = EFindSqlStateType.stnormal;
                  gcurrentsqlstatement.addtokentolist(ast);
                  gcurrentsqlstatement.semicolonended = ast;
                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                  continue;
              }

              if (ast.tokencode  == TBaseType.cmtdoublehyphen){
                  if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                      gst = EFindSqlStateType.stnormal;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                      continue;
                  }
              }

              gcurrentsqlstatement.addtokentolist(ast);

              break;
          }
          case ststoredprocedure :
          {

              boolean readyToEnd = true;
              if (ast.tokencode == ';'){
                  // don't count case..end, sample ... end after ;
                  lcNestedSample = 0;
                  lcNestedCase = 0;
              }else if (ast.tokencode == TBaseType.rrw_case){ // case ... end
                  lcNestedCase++;

                  TSourceToken prevst = ast.prevSolidToken();
                  if (prevst != null){
                    if (prevst.tokencode == TBaseType.rrw_end){
                        // don't count:  end case
                        lcNestedCase--;
                    }
                  }

              }else if (ast.tokencode == TBaseType.rrw_teradata_sample){ // sample ... end
                  lcNestedSample++;
              }

              if ((gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatetrigger)
                  ||(gcurrentsqlstatement.sqlstatementtype == sstcreateprocedure)
                      ||(gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatefunction)
              )
              {
                  if ((ast.tokencode == '(')||(ast.tokencode == TBaseType.left_parenthesis_2)) lcNestedParens++;
                  if (ast.tokencode == ')') {
                      lcNestedParens--;
                      if (lcNestedParens < 0) lcNestedParens = 0;
                  }

                  if (ast.tokencode == TBaseType.rrw_begin) lcNestedBeginEnd++;
                  if (ast.tokencode == TBaseType.rrw_end) {


                      boolean countThisEnd = true;
                      if (lcNestedSample>0){
                          // don't count sample...end
                          lcNestedSample--;
                          countThisEnd = false;
                      }else if (lcNestedCase>0){
                          // don't count case...end
                          lcNestedCase--;
                          countThisEnd = false;
                      }

                      if (countThisEnd){
                          // don't count end while/if/case/loop/for/repeat
                          TSourceToken nt = ast.nextSolidToken();
                          if (nt != null){
                              countThisEnd = ((nt.tokencode != TBaseType.rrw_while)
                                      &&(nt.tokencode != TBaseType.rrw_if)&&(nt.tokencode != TBaseType.rrw_case)
                                      &&(nt.tokencode != TBaseType.rrw_loop)&&(nt.tokencode != TBaseType.rrw_for)
                                      &&(nt.tokencode != TBaseType.rrw_repeat)
                              );
                          }
                      }

                      if (countThisEnd){
                          lcNestedBeginEnd--;
                          if (lcNestedBeginEnd < 0 ) lcNestedBeginEnd = 0;
                      }
                  }

                  readyToEnd = ((lcNestedParens == 0)&&(lcNestedBeginEnd == 0));
              }
              // single stmt in function/procedure/trigger may use ; as terminate char
              // so default terminate char is ;, if begin is found ,then
              //set terminate char to DelimiterChar
                if (curdelimiterchar !=  delimiterchar){
                  if (ast.tokencode == TBaseType.rrw_begin )
                    {curdelimiterchar = delimiterchar;}
                  else if (ast.tokencode == TBaseType.mslabel ){
                      if (ast.container.nextsolidtoken(ast,1,false).tokencode == TBaseType.rrw_begin ){
//                          if (ast.container.solidtokenafterpos(ast.posinlist,TBaseType.rrw_begin,1,"") == 1) {
                          curdelimiterchar = delimiterchar;
                      }
                  }
                }

                    if (curdelimiterchar == ';')
                    {
                        gcurrentsqlstatement.addtokentolist(ast);
                        if ((ast.tokentype == ETokenType.ttsemicolon)&(gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstcreatetrigger)&(readyToEnd))
                        {
                            gst = EFindSqlStateType.stnormal;
                            //asqlstatement.SemiColonEnded := ast;
                             gcurrentsqlstatement._semicolon = ast;
                            doongetrawsqlstatementevent(gcurrentsqlstatement);
                            continue;
                        }
                    }else
                    {
                          if (ast.astext.length() == 1 )
                          { ch = ast.astext.charAt(0);}
                          else if (ast.astext.length() > 1)
                           { ch = ast.astext.charAt(ast.astext.length()-1); }
                          else
                          { ch = ' ';}


                          if (((ast.tokencode != TBaseType.cmtslashstar)&&(ast.tokencode != TBaseType.cmtdoublehyphen))
                                  && ((ch == curdelimiterchar )&&(ast.isFirstTokenOfLine())&&(readyToEnd)))
                          {
                              if (ast.astext.length() > 1)
                              {
                                  lcstr = ast.astext.substring(0,ast.astext.length()-1);
                                  if ((c=flexer.getkeywordvalue(lcstr))>0 )
                                    ast.tokencode = c;
                              }
                              else
                              {
                                 // ast.TokenStatus := tsIgnoreByyacc;
                                 // ast.Tokentype :=   ttDoublehyphenComment; //make this tokentext available like a comment
                                  gcurrentsqlstatement._semicolon = ast;
                              }
                              gcurrentsqlstatement.addtokentolist(ast);
                              gst = EFindSqlStateType.stnormal;
                              doongetrawsqlstatementevent(gcurrentsqlstatement);
                          }else if ((ast.searchToken(TBaseType.rrw_create,1) != null)&&(ast.searchToken(TBaseType.rrw_procedure,4) != null)&&(readyToEnd))
                          {
                              // gcurrentsqlstatement.addtokentolist(ast);
                              gst = EFindSqlStateType.stnormal;
                              doongetrawsqlstatementevent(gcurrentsqlstatement);

                          }else if (ast.tokencode == ';'){
                              gcurrentsqlstatement.addtokentolist(ast);

                              boolean startNewStatement = false;
                              if (readyToEnd){
                                  startNewStatement  = (sqlcmds.issql(ast.nextSolidToken(), dbVendor,gst,gcurrentsqlstatement) != null);
                              }
                              if (startNewStatement){
                                  gst = EFindSqlStateType.stnormal;
                                  doongetrawsqlstatementevent(gcurrentsqlstatement);
                              }
                          }
                          else{
                              gcurrentsqlstatement.addtokentolist(ast);
                          }
                    } // end of DelimiterChar

                if (( gst == EFindSqlStateType.ststoredprocedure ) && (ast.tokencode  == TBaseType.cmtdoublehyphen)){
                  if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)){ // -- sqlflow-delimiter
                      gst = EFindSqlStateType.stnormal;
                      doongetrawsqlstatementevent(gcurrentsqlstatement);
                  }
                }

              break;
          } //teradata sp

        } //case
    } //for


    //last statement
    if (TBaseType.assigned(gcurrentsqlstatement) && ( (gst == EFindSqlStateType.stsql) ||(gst == EFindSqlStateType.stBTEQCmd) || (gst == EFindSqlStateType.ststoredprocedure) || (gst == EFindSqlStateType.sterror)) )
    {
        doongetrawsqlstatementevent(gcurrentsqlstatement);
    }

    return errorcount;//FSqlStatements.Count;

}

int dogetrawsqlstatements(){

    sqlstatements.clear();
    if (sourcetokenlist.size() == 0) return -1;

    switch(dbVendor){
        case dbvmssql:
        case dbvazuresql: {
            return domssqlgetrawsqlstatements();
        }
        case dbvsybase:{
            return dosybasegetrawsqlstatements();
        }
        case dbvinformix:{
            return doinformixgetrawsqlstatements();
        }
        case dbvoracle:{
            return dooraclegetrawsqlstatements();
        }
        case dbvdb2:{
            return dodb2getrawsqlstatements();
        }
        case dbvmysql:{
            return domysqlgetrawsqlstatements();
        }
        case dbvteradata:{
            return doteradatagetrawsqlstatements();
        }
        case dbvpostgresql:{
            return dopostgresqlgetrawsqlstatements();
        }
        case dbvredshift:{
            return doredshiftgetrawsqlstatements();
        }
        case dbvgreenplum:{
            return dogreenplumgetrawsqlstatements();
        }
        case dbvmdx:{
            return domdxgetrawsqlstatements();
        }
        case dbvnetezza:{
            return donetezzagetrawsqlstatements();
        }
        case dbvhive:{
            return dohivegetrawsqlstatements();
        }
        case dbvimpala:{
            return doimpalagetrawsqlstatements();
        }
        case dbvhana:{
            return dohanagetrawsqlstatements();
        }
        case dbvdax:{
            return dodaxgetrawsqlstatements();
        }
        case dbvodbc:{
            return doodbcgetrawsqlstatements();
        }
        case dbvvertica:{
            return doverticagetrawsqlstatements();
        }
        case dbvopenedge:{
            return domssqlgetrawsqlstatements();
        }
        case dbvcouchbase:{
            return docouchbasegetrawsqlstatements();
        }
        case dbvsnowflake:{
            int i = dosnowflakegetrawsqlstatements();
            // expland $$ string into token list
            expandDollarString();
            return i;
        }
        case dbvbigquery:{
            return dobigquerygetrawsqlstatements();
        }
        case dbvsoql:{
            return dosoqlgetrawsqlstatements();
        }
        case dbvsparksql:{
            return dosparksqlgetrawsqlstatements();
        }
        case dbvpresto:{
            return doprestogetrawsqlstatements();
        }
        case dbvathena:{
            return doprestogetrawsqlstatements();
        }
        case dbvdatabricks:{
            return dodatabricksgetrawsqlstatements();
        }
        case dbvgaussdb:{
            return dogaussdbgetrawsqlstatements();
        }
        case dbvansi:{
            return doansigetrawsqlstatements();
        }
        default:{
            return domssqlgetrawsqlstatements();
        }
    }

}

    /**
     *  separates the SQL statements in the input SQL script without doing syntax check.
     *  <p></p>
     *  Use the {@link #getSqlstatements()} method to get the list of SQL statements.
     *  The SQL statement object is the instance of the sub-class of {@link TCustomSqlStatement}, get SQL statement type
     *  via the {@link TCustomSqlStatement#sqlstatementtype} field, get string representation of
     *  each SQL statement via the {@link TCustomSqlStatement#toString} method.
     *  <p></p>
     *  All source tokens in this SQL statement
     *  is available by using {@link TCustomSqlStatement#sourcetokenlist} filed.
     *  Since no parse tree is built by calling this method, no further detailed information about the SQL statement is available.
     *
      * @return 0 if get SQL statements successfully
     */
public int getrawsqlstatements(){
    
    int ret = readsql();
    if (ret != 0) return ret;
    dosqltexttotokenlist();

    return dogetrawsqlstatements();
}

    /**
     *  turns the input SQL into a sequence of token which is the
     *  basic lexis element of SQL syntax. Token is categorized as keyword, identifier,
     *  number, operator, whitespace and other types. All source tokens can be fetched
     *  via the {@link #getSourcetokenlist()} method.
     *
     */
public void tokenizeSqltext(){
    readsql();
    dosqltexttotokenlist();
}



void findAllSyntaxErrorsInPlsql(TCustomSqlStatement psql){
    if (psql.getErrorCount() > 0){
        copyerrormsg(psql);
    }

    for (int k=0;k<psql.getStatements().size();k++){
        findAllSyntaxErrorsInPlsql(psql.getStatements().get(k));
    }

}

private TSQLEnv sqlEnv = null;

    /**
     * SQL environment includes the database metadata such as procedure, function, trigger, table and etc.
     *
     * In order to link column to table correctly without connecting to database,
     * we need to provide a class which implements {@link TSQLEnv} to TGSqlParser.
     * this class tells TGSqlParser the relationship between column and table.
     *
     * <p>Take this SQL for example:
     * <pre>
     * SELECT Quantity,b.Time,c.Description
     * FROM
     * (SELECT ID2,Time FROM bTab) b
     * INNER JOIN aTab a on a.ID=b.ID
     * INNER JOIN cTab c on a.ID=c.ID
     * </pre>
     *
     * <p>General SQL Parser can build relationship between column: ID2 and table: bTable
     * correctly without metadata information from database because there is only one table
     * in from clause. But it can't judge column: Quantity belong to table: aTab or cTab,
     * since no table alias was prefixed to column: Quantity. If no metadata provided,
     * General SQL Parser will link column: Quantity to the first valid table (here it is aTab)
     *
     * <p>If we create a class  TRealDatabaseSQLEnv implements {@link TSQLEnv},then
     *  {@link #setSqlEnv(TSQLEnv)}, General SQL Parser can take this advantage to create
     *  a correct relationship between column and tables.
     *
     *<pre>
     * class TSQLServerEnv extends TSQLEnv{
     *
     *  public TSQLServerEnv(){
     *          super(EDbVendor.dbvmssql);
     *          initSQLEnv();
     *        }
     *
     *    &#64;Override
     *    public void initSQLEnv() {
     *
     *          // add a new database: master
     *          TSQLCatalog sqlCatalog = createSQLCatalog("master");
     *          // add a new schema: dbo
     *          TSQLSchema sqlSchema = sqlCatalog.createSchema("dbo");
     *          //add a new table: aTab
     *          TSQLTable aTab = sqlSchema.createTable("aTab");
     *          aTab.addColumn("Quantity1");
     *
     *          //add a new table: bTab
     *          TSQLTable bTab = sqlSchema.createTable("bTab");
     *          bTab.addColumn("Quantity2");
     *
     *          //add a new table: cTab
     *          TSQLTable cTab = sqlSchema.createTable("cTab");
     *          cTab.addColumn("Quantity");
     *
     *    }
     * }
     * </pre>
     *
     * @return SQL environment
     */
    public TSQLEnv getSqlEnv() {
        return sqlEnv;
    }

    private boolean onlyNeedRawParseTree = false;

    public void setOnlyNeedRawParseTree(boolean onlyNeedRawParseTree) {
        this.onlyNeedRawParseTree = onlyNeedRawParseTree;
    }

    public void setSqlEnv(TSQLEnv sqlEnv) {
        this.sqlEnv = sqlEnv;
    }


     // Time tracking variables
     private boolean enableTimeLogging = false;
     private long rawSqlStatementsTime = 0;
     private long parsingTime = 0;
     private long semanticAnalysisTime = 0;
     private long interpreterTime = 0;
     
     /**
      * Enable or disable time logging for parser steps
      * @param enable true to enable time logging, false to disable
      */
     public void setEnableTimeLogging(boolean enable) {
         this.enableTimeLogging = enable;
     }
     
     /**
      * Check if time logging is enabled
      * @return true if time logging is enabled, false otherwise
      */
     public boolean isTimeLoggingEnabled() {
         return this.enableTimeLogging;
     }
     
     /**
      * Reset all accumulated time counters to zero
      */
     public void resetTimeCounters() {
         rawSqlStatementsTime = 0;
         parsingTime = 0;
         semanticAnalysisTime = 0;
         interpreterTime = 0;
     }
     
     /**
      * Get accumulated time spent getting raw SQL statements in milliseconds
      * @return time in milliseconds
      */
     public long getRawSqlStatementsTime() {
         return rawSqlStatementsTime;
     }
     
     /**
      * Get accumulated time spent parsing in milliseconds
      * @return time in milliseconds
      */
     public long getParsingTime() {
         return parsingTime;
     }
     
     /**
      * Get accumulated time spent on semantic analysis in milliseconds
      * @return time in milliseconds
      */
     public long getSemanticAnalysisTime() {
         return semanticAnalysisTime;
     }
     
     /**
      * Get accumulated time spent in interpreter in milliseconds
      * @return time in milliseconds
      */
     public long getInterpreterTime() {
         return interpreterTime;
     }
     
     /**
      * Get total accumulated time spent in all steps
      * @return total time in milliseconds
      */
     public long getTotalTime() {
         return rawSqlStatementsTime + parsingTime + semanticAnalysisTime + interpreterTime;
     }
     
     /**
      * Returns time statistics as a formatted string
      * @return formatted string with time statistics
      */
     public String getTimeStatistics() {
         if (!enableTimeLogging) {
             return "Time logging is disabled";
         }
         
         StringBuilder sb = new StringBuilder();
         sb.append(String.format("Time statistics for TGSqlParser, dbvendor: %s:\n",this.dbVendor));
         sb.append(String.format("1. Raw SQL statements: %d ms (%.2f%%)\n", 
                 rawSqlStatementsTime,
                 getTotalTime() > 0 ? 100.0 * rawSqlStatementsTime / getTotalTime() : 0));
         sb.append(String.format("2. Parsing: %d ms (%.2f%%)\n", 
                 parsingTime,
                 getTotalTime() > 0 ? 100.0 * parsingTime / getTotalTime() : 0));
         sb.append(String.format("3. Semantic analysis: %d ms (%.2f%%)\n", 
                 semanticAnalysisTime,
                 getTotalTime() > 0 ? 100.0 * semanticAnalysisTime / getTotalTime() : 0));
         sb.append(String.format("4. Interpreter: %d ms (%.2f%%)\n", 
                 interpreterTime,
                 getTotalTime() > 0 ? 100.0 * interpreterTime / getTotalTime() : 0));
         sb.append(String.format("Total: %d ms", getTotalTime()));
         return sb.toString();
     }
     
     int doparse() {
         int j;
         long startTime, endTime;
 
         // 1. start get raw sql statements
         startTime = enableTimeLogging ? System.currentTimeMillis() : 0;
         
         int ret = getrawsqlstatements();
         
         if (enableTimeLogging) {
             endTime = System.currentTimeMillis();
             rawSqlStatementsTime += (endTime - startTime);
         }
 
         boolean isPushGloablStack = false;
         //if (ret != 0) return ret;
         TFrame firstFrame = null;
 
         globalContext = new TContext();
 
         if (this.sqlEnv == null) {
             this.sqlEnv = new TSQLEnv(this.dbVendor) {
                 @Override
                 public void initSQLEnv() {
                 }
             };
             // this.sqlEnv.setEnableGetMetadataFromDDL(false);
         }
         globalContext.setSqlEnv(this.sqlEnv, this.sqlstatements);
 
         //TStackFrame stackFrame = new TStackFrame(new TGlobalScope());
         if (getFrameStack().size() == 0) { // stack passed from outside gsqlparser will contain frames
             //getFrameStack().push(new TStackFrame(new TGlobalScope()));
             TGlobalScope globalScope = new TGlobalScope();
 
             globalScope.setSqlEnv(this.sqlEnv);
             firstFrame = new TFrame(globalScope);
             firstFrame.pushMeToStack(getFrameStack());
             isPushGloablStack = true;
         }
 
         // 2. start parsing
         startTime = enableTimeLogging ? System.currentTimeMillis() : 0;
         
         for (int i = 0; i < sqlstatements.size(); i++) {
             sqlstatements.getRawSql(i).setFrameStack(frameStack);
             j = sqlstatements.getRawSql(i).parsestatement(null, false, onlyNeedRawParseTree);
 
             TCustomSqlStatement sql0 = null;
             if (sqlstatements.get(i).isoracleplsql()) {
                 // check syntax error in select/insert statement inside plsql
                 sql0 = sqlstatements.get(i);
                 findAllSyntaxErrorsInPlsql(sql0);
             }
 
             boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
 
             if (doRecover && ((j != 0) || (sqlstatements.get(i).getErrorCount() > 0))) {
                 if (((sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreatetable)
                         || ((sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreateindex) && (dbVendor != EDbVendor.dbvcouchbase))
                         ) && (!TBaseType.c_createTableStrictParsing)
                 ) {
                     // only parse main body of create table,
                     TCustomSqlStatement errorSqlStatement = (TCustomSqlStatement) sqlstatements.get(i);
 
                     int nested = 0;
                     boolean isIgnore = false, isFoundIgnoreToken = false;
                     TSourceToken firstIgnoreToken = null;
                     for (int k = 0; k < errorSqlStatement.sourcetokenlist.size(); k++) {
                         TSourceToken st = errorSqlStatement.sourcetokenlist.get(k);
                         if (isIgnore) {
                             if (st.issolidtoken() && (st.tokencode != ';')) {
                                 isFoundIgnoreToken = true;
                                 if (firstIgnoreToken == null) {
                                     firstIgnoreToken = st;
                                 }
                             }
                             if (st.tokencode != ';') {
                                 st.tokencode = TBaseType.sqlpluscmd;
                             }
                             continue;
                         }
                         if (st.tokencode == (int) ')') {
                             nested--;
                             if (nested == 0) {
                                 //let's check is next token is
                                 // as ( select
                                 boolean isSelect = false;
                                 TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
                                 if (st1 != null) {
                                     TSourceToken st2 = st.searchToken((int) '(', 2);
                                     if (st2 != null) {
                                         TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
                                         isSelect = (st3 != null);
                                     }
                                 }
                                 if (!isSelect) isIgnore = true;
                             }
                         }
                         if ((st.tokencode == (int) '(') || (st.tokencode == TBaseType.left_parenthesis_2)) {
                             nested++;
                         }
                     }
 
                     if ((dbVendor == EDbVendor.dbvoracle) && ((firstIgnoreToken != null) && (!TBaseType.searchOracleTablePros(firstIgnoreToken.toString())))) {
                         // if it is not the valid Oracle table properties option, let raise the error.
                         isFoundIgnoreToken = false;
                     }
                     if (isFoundIgnoreToken) {
                         errorSqlStatement.clearError();
                         j = sqlstatements.get(i).parsestatement(null, false);
                     }
                 }
 
                 if (((sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreatetrigger)
                         || (sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstcreatefunction)
                         || (sqlstatements.get(i).sqlstatementtype == sstcreateprocedure)
                         ) && (dbVendor == EDbVendor.dbvdb2)) { 
                     // db2 can handle Oracle pl/sql code, so we give it a try here
                     TCustomSqlStatement stmt = sqlstatements.get(i);
                     StringBuffer stmtStr = new StringBuffer(1024);
                     for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
                         stmtStr.append(stmt.sourcetokenlist.get(k).astext);
                     }
 
                     TGSqlParser lc_sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
                     lc_sqlparser.sqltext = stmtStr.toString();
                     int iRet = lc_sqlparser.parse();
                     if (iRet == 0) {
                         sqlstatements.remove(i);
                         sqlstatements.add(i, lc_sqlparser.sqlstatements.get(0));
                         continue;
                     }
                 }
             }
 
             if ((j != 0) || (sqlstatements.get(i).getErrorCount() > 0)) {
                 copyerrormsg(sqlstatements.get(i));
 
                 if ((isEnablePartialParsing()) && (dbVendor == EDbVendor.dbvsybase) && (sqlstatements.get(i).sqlstatementtype == ESqlStatementType.sstmssqlcreateprocedure)) {
                     TMssqlCreateProcedure createProcedure = (TMssqlCreateProcedure) sqlstatements.get(i);
 
                     StringBuffer storedProcedure = new StringBuffer(1024);
                     boolean ASKeyword = false;
                     for (int k = 0; k < createProcedure.sourcetokenlist.size(); k++) {
                         if ((!ASKeyword) && (createProcedure.sourcetokenlist.get(k).tokencode == TBaseType.rrw_as)) {
                             ASKeyword = true;
                             continue;
                         }
                         if (ASKeyword) {
                             storedProcedure.append(createProcedure.sourcetokenlist.get(k).astext);
                         }
                     }
 
                     TGSqlParser lc_sqlparser = new TGSqlParser(dbVendor);
                     lc_sqlparser.sqltext = storedProcedure.toString();
                     lc_sqlparser.parse();
                     for (int k = 0; k < lc_sqlparser.sqlstatements.size(); k++) {
                         createProcedure.getBodyStatements().add(lc_sqlparser.sqlstatements.get(k));
                     }
                 }
             }
 
             // fire SQL Statement handle event if set, if return true, stop parsing
             if (sqlStatementHandle != null) {
                 if (sqlStatementHandle.processSQLStatement(sqlstatements.get(i), this)) break;
             }
         }
         
         if (enableTimeLogging) {
             endTime = System.currentTimeMillis();
             parsingTime += (endTime - startTime);
         }
 
         if (isPushGloablStack) {
             firstFrame.popMeFromStack(getFrameStack());
         }
 
         // 3. start semantic analysis
         startTime = enableTimeLogging ? System.currentTimeMillis() : 0;
         
         if ((TBaseType.ENABLE_RESOLVER) && (getErrorCount() == 0)) {
             TSQLResolver resolver = new TSQLResolver(globalContext, sqlstatements);
             if (!resolver.resolve()) {
                 // Handle resolution errors
                 // addErrors(resolver.getLog().getErrors());
             }
         }
         
         if (enableTimeLogging) {
             endTime = System.currentTimeMillis();
             semanticAnalysisTime += (endTime - startTime);
         }
 
         // 4. start interpreter
         startTime = enableTimeLogging ? System.currentTimeMillis() : 0;
         
         if ((TBaseType.ENABLE_INTERPRETER) && (getErrorCount() == 0)) {
             TLog.clearLogs();
             TGlobalScope globalScope = new TGlobalScope(sqlEnv);
             TLog.enableInterpreterLogOnly();
 
             TASTEvaluator astEvaluator = new TASTEvaluator(this.sqlstatements, globalScope);
             astEvaluator.eval();
         }
         
         if (enableTimeLogging) {
             endTime = System.currentTimeMillis();
             interpreterTime += (endTime - startTime);
         }
 
         return getErrorCount();
     }

     
void copyerrormsg(TCustomSqlStatement sql){
    for (int i = 0; i<sql.getSyntaxErrors().size(); i++){
            this.syntaxErrors.add(new TSyntaxError( (TSyntaxError)sql.getSyntaxErrors().get(i) ));
    }
//    for (int i = 0; i<sql.getSyntaxHints().size(); i++){
//            this.syntaxHints.add(new TSyntaxError( (TSyntaxError)sql.getSyntaxHints().get(i) ));
//    }
}

private static String calculateLicenseKey(boolean ignoreMachineId){

    if (userName == null) return null;
    if (machineId == null) return null;

    byte[] bytesOfMessage=null;
    String licenseStr = "I love sql pretty printer, yeah!"+userName.toLowerCase();
    if (!ignoreMachineId){
        licenseStr += machineId.toLowerCase();
    }
    try {
        bytesOfMessage = licenseStr.getBytes("UTF-8");
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    }

    MessageDigest md = null;
    try {
        md = MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
        e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
    }
    byte[] digest = md.digest(bytesOfMessage);

    return null;//HardwareBinder.getHex(digest);
}

private static boolean validateLicense(){

    int ret = 0;
//    String line;
//
//    try{
//        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerTeradata.class.getResourceAsStream(TBaseType.license_file_name)));
//        line = br.readLine().trim();
//        String[] ss = line.split("[|]");
//        if (ss.length != 3){
//            licenseMessage = TBaseType.license_file_invalid;
//            ret = -2;
//        }else {
//            licenseType = ss[0].trim();
//            userName = ss[1].trim();
//            licenseKey = ss[2].trim();
//        }
//    }catch (NullPointerException n){
//        licenseMessage = TBaseType.license_file_missing;
//        ret = -1;
//    }catch(IOException e){
//        licenseMessage = TBaseType.license_file_missing;
//        ret = -1;
//    }
//
//    machineId = new HardwareBinder().getMachineIdString();
//
//    if (ret == 0 ){
//        boolean ignoreMachineId = licenseType.equalsIgnoreCase(TBaseType.license_type_dist);
//        if (!calculateLicenseKey(ignoreMachineId).equalsIgnoreCase(licenseKey)){
//            licenseMessage = TBaseType.license_illegal_key;
//            ret = -3;
//        }
//    }else {
//        licenseMessage += ", please contact "+TBaseType.vendorEmail +" to get a license file by providing this number[UUID]"+TBaseType.newline+machineId;
//    }

    return ret == 0;

}

private static boolean check_license_time(){

    boolean  ret = false;

    String toDate = TBaseType.license_expired_date;

    DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT);
    Calendar currDtCal = Calendar.getInstance();

    // Zero out the hour, minute, second, and millisecond
    currDtCal.set(Calendar.HOUR_OF_DAY, 0);
    currDtCal.set(Calendar.MINUTE, 0);
    currDtCal.set(Calendar.SECOND, 0);
    currDtCal.set(Calendar.MILLISECOND, 0);

    Date currDt = currDtCal.getTime();

    Date toDt;
    try {
        toDt = df.parse(toDate);
    } catch (ParseException e) {
        toDt = null;
        // Print some error message back to the user
    }

    if (toDt != null){
        int results = toDt.compareTo(currDt);
        ret = results > 0;
    }

    return ret;
}

    public void setTeradataUtilityType(TeradataUtilityType teradataUtilityType) {
        if ((this.getFlexer() != null) && (this.getFlexer() instanceof TLexerTeradata)){
            ((TLexerTeradata)this.getFlexer()).setTeradataUtilityType(teradataUtilityType);
        }
    }

}