package gudusoft.gsqlparser;


import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;

/**
 * Base lexer of all databases - Core tokenization engine for SQL parsing.
 * 
 * The lexer reads SQL text character by character and produces tokens that represent
 * the syntactic units of SQL. This process involves several key components and stages:
 * 
 * <h3>1. Input Management and Buffering</h3>
 * <ul>
 *   <li><b>yyinput (BufferedReader)</b>: Primary input source for SQL text</li>
 *   <li><b>yyline (char[])</b>: Current line buffer read from input via readln()</li>
 *   <li><b>buf (char[])</b>: Reversed line buffer for character-by-character processing</li>
 *   <li><b>bufptr</b>: Current position in buf, decrements as characters are consumed</li>
 * </ul>
 * 
 * <h3>2. Token Text Formation Process</h3>
 * <pre>
 * SQL Input → readln() → yyline[] → reversed into buf[] → get_char() → yytextbuf[]
 *                                                                        ↓
 *                                                                yylex() processing
 *                                                                        ↓
 *                                                                 yylvalstr (String)
 *                                                                        ↓
 *                                                            TSourceToken.astext
 * </pre>
 * 
 * <h4>Key Variables in Token Text Storage:</h4>
 * <ul>
 *   <li><b>yytextbuf (char[])</b>: Accumulator buffer for current token being formed</li>
 *   <li><b>yytextlen</b>: Current length of text in yytextbuf</li>
 *   <li><b>yytextbufsize</b>: Allocated size of yytextbuf (dynamically grows)</li>
 *   <li><b>yylvalstr (String)</b>: Final token text string created from yytextbuf</li>
 *   <li><b>literalbuf (StringBuilder)</b>: Special buffer for string literals and complex tokens</li>
 * </ul>
 * 
 * <h3>3. Position Tracking System</h3>
 * 
 * The lexer maintains precise position information for every token:
 * <ul>
 *   <li><b>yylineno</b>: Current line number (1-based)</li>
 *   <li><b>yycolno</b>: Current column number (0-based)</li>
 *   <li><b>offset</b>: Absolute character offset from start of input</li>
 *   <li><b>yylineno_p, yycolno_p, offset_p</b>: Previous position values for token start</li>
 * </ul>
 * 
 * <h3>4. Token Creation Workflow</h3>
 * <ol>
 *   <li>Characters are read via get_char() from buf[] into yytextbuf[]</li>
 *   <li>yylex() identifies token boundaries and type</li>
 *   <li>Token text is extracted: yylvalstr = new String(yytextbuf, 0, yytextlen)</li>
 *   <li>yylexwrap() creates TSourceToken with:
 *       <ul>
 *         <li>astext = yylvalstr (full token text copy)</li>
 *         <li>lineNo = yylineno_p (start line)</li>
 *         <li>columnNo = yycolno_p (start column)</li>
 *         <li>offset = offset_p (absolute position)</li>
 *       </ul>
 *   </li>
 * </ol>
 * 
 * <h3>5. Memory Management and Text Copying</h3>
 * 
 * <b>Current Implementation (Eager Loading):</b>
 * <ul>
 *   <li>Every token immediately copies its text from yytextbuf to TSourceToken.astext</li>
 *   <li>Original SQL text in yyline is discarded after processing each line</li>
 *   <li>No direct link maintained between token and original input position</li>
 * </ul>
 * 
 * <h3>6. Tracing Back to Original Position</h3>
 * 
 * <b>Currently Possible:</b>
 * <ul>
 *   <li>Token stores lineNo, columnNo, and offset</li>
 *   <li>These can theoretically locate position in original input</li>
 * </ul>
 * 
 * <b>Current Limitations:</b>
 * <ul>
 *   <li>Original input text is not retained after line processing</li>
 *   <li>yyline buffer is overwritten for each new line</li>
 *   <li>No mechanism to retrieve original text from position alone</li>
 * </ul>
 * 
 * @author Gudu Software
 */
public class TCustomLexer {

    // 在 lexer level 创建 token table, 按照 token code存储所有 token 的一些关键信息，主要用于处理一个关键字token被用作column，table name的情况
    public static int MAX_TOKEN_SIZE = 2048; // 所有可能的token的数量
    public static int MAX_TOKEN_COLUMN_SIZE = 10;

    // 定义一个具有 MAX_TOKEN_SIZE 个元素的常量数组，每个元素有 MAX_TOKEN_COLUMN_SIZE 列， 列的类型为整数
    // column 0: 代表该token出现的次数
    // column 1: 代表该token第一次出现的 x position
    // column 2: 代表该token第一次出现的 y position
    // column 3: 代表该token最后一次出现的 x position
    // column 4: 代表该token最后一次出现的 y position
    // column 5: 代表该token第一次出现的 position in the token list
    // column 6: 代表该token最后一次出现的 position in the token list

    public static int COLUMN0_COUNT = 0;
    public static int COLUMN1_FIRST_X = 1;
    public static int COLUMN2_FIRST_Y = 2;
    public static int COLUMN3_LAST_X = 3;
    public static int COLUMN4_LAST_Y = 4;
    public static int COLUMN5_FIRST_POS = 5;
    public static int COLUMN6_LAST_POS = 6;

    /**
     * Pre-allocated strings for single ASCII characters (0-127).
     * Used to avoid creating new String objects for common single-char tokens
     * like '(', ')', ',', ';', '+', '-', '*', '/', etc.
     * This significantly reduces GC pressure in the lexer hot path.
     */
    private static final String[] SINGLE_CHAR_STRINGS = new String[128];
    static {
        for (int i = 0; i < 128; i++) {
            SINGLE_CHAR_STRINGS[i] = String.valueOf((char) i);
        }
    }

    public long[][] TOKEN_TABLE = new long[MAX_TOKEN_SIZE][MAX_TOKEN_COLUMN_SIZE];

    /**
     * Tracks which tokenIds have been written to TOKEN_TABLE during current parse.
     * Used for incremental reset - only clear entries that were actually used.
     */
    private int[] usedTokenIds = new int[512];  // Typical SQL uses <200 distinct token types
    private int usedTokenCount = 0;

    /**
     * Reset TOKEN_TABLE by only clearing entries that were used (incremental clear).
     * This is O(usedTokenCount) instead of O(MAX_TOKEN_SIZE * MAX_TOKEN_COLUMN_SIZE).
     * For typical SQL with ~100 distinct token types, this saves clearing ~20,000 entries.
     */
    public void resetTokenTable() {
        for (int i = 0; i < usedTokenCount; i++) {
            int tokenId = usedTokenIds[i];
            for (int j = 0; j < MAX_TOKEN_COLUMN_SIZE; j++) {
                TOKEN_TABLE[tokenId][j] = 0L;
            }
        }
        usedTokenCount = 0;
    }

    // define a function to set value when token is found, input is token id, a token with TSourceToken type
    public void setTokenTableValue( TSourceToken token) {
        if (token == null) return;
        int tokenId = token.tokencode;

        if (tokenId < 0 || tokenId >= MAX_TOKEN_SIZE) {
            return;
        }
        if (TOKEN_TABLE[tokenId][COLUMN0_COUNT] == 0) {
            // Track this tokenId for incremental reset
            if (usedTokenCount < usedTokenIds.length) {
                usedTokenIds[usedTokenCount++] = tokenId;
            }
            TOKEN_TABLE[tokenId][COLUMN0_COUNT] = 1;
            TOKEN_TABLE[tokenId][COLUMN1_FIRST_X] = token.lineNo;
            TOKEN_TABLE[tokenId][COLUMN2_FIRST_Y] = token.columnNo;
            TOKEN_TABLE[tokenId][COLUMN3_LAST_X] = token.lineNo;
            TOKEN_TABLE[tokenId][COLUMN4_LAST_Y] = token.columnNo;
            TOKEN_TABLE[tokenId][COLUMN5_FIRST_POS] = token.posinlist;
            TOKEN_TABLE[tokenId][COLUMN6_LAST_POS] = token.posinlist;
        } else {
            TOKEN_TABLE[tokenId][COLUMN0_COUNT] += 1;
            TOKEN_TABLE[tokenId][COLUMN3_LAST_X] = token.lineNo;
            TOKEN_TABLE[tokenId][COLUMN4_LAST_Y] = token.columnNo;
            TOKEN_TABLE[tokenId][COLUMN6_LAST_POS] = token.posinlist;
        }
    }

    public BufferedReader yyinput;
    long yylineno,yycolno,offset,yylineno_p,yycolno_p,offset_p;
    int bufptr,yystate,yysstate,yylstate,yytextlen,yyretval, yytextbufsize,
            yymatches,yysleng;
    char[] yyline;
    /**
     * Reusable buffer for readln() to reduce per-line allocations.
     * Expands as needed for long lines and stays expanded for reuse.
     */
    private char[] lineReadBuffer = new char[4096];
    /**
     * Actual content length in lineReadBuffer/yyline.
     * Used instead of yyline.length since lineReadBuffer is reused without copying.
     * Like yyline, it still describes the last line read once the input is exhausted.
     */
    private int yylineLen;
    String yylvalstr;
    public String  dolqstart = "";//postgresql, start part of Dollar-quoted String Constants
    char yylastchar,yyactchar,yytablechar;
    boolean yydone,yyreject;
    char[] yytextbuf;
    char[] buf;
    int bufsize;
    boolean endOfInput;

    //StringBuffer literalbuf;
    StringBuilder literalbuf;
    int literallen,literalalloc,xcdepth,nchars,slashstar,dashdash;
    boolean isqmarktoident;
    public boolean insqlpluscmd;
    char dummych1,dummych2,dummych3;
    boolean utf8NoBreakSpaceReady = false;

    int nestedLessThan = 0;

    boolean isReadyForFunctionBody = false, isInFunctionBody = false;
    int   functionBodyDelimiterIndex = -1;
    ArrayList<String> functionBodyDelimiter = new ArrayList<>();

    public static int keyword_type_reserved = 0x0001;
    public static int keyword_type_keyword = 0x0002;
    public static int keyword_type_identifier = 0x0004;
    public static int keyword_type_column = 0x0008;

    public char delimiterchar;
    public String defaultDelimiterStr;
    public String tmpDelimiter;
    
    final static int intial_bufsize = 16384;
    final static char lf = (char)10;
    final static int max_chars = 65536*10*2;
    final static int max_rules = 256*2*10;
    int  max_matches = 1024*20*10*2;


    // 下面这些常量按照在 l 文件中出现的次序，必须以  +2 的方式递加. 为什么以 +2 的方式递加 原因忘了，尚未搞清楚。
    final static int init = 2;
    final static int xc = 4;
    final static int xd = 6;
    final static int xq = 8;
    final static int xqq = 10;  //oracle
    final static int xdolq = 10;//postgresql
    final static int xdbracket = 10;
    final static int xdbrace = 12;
    final static int xbacktick = 12;

    final static int xbracketrs = 12; //redshift
    final static int xdolq_clickhouse = 14;//clickhouse: 7th %start in its .l, after xdbracket(10) and xbacktick(12)
    final static int xqtriple = 14;//bigquery
    final static int xdtriple = 16;//bigquery



    //https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server
    final static int namedatalen = 8060;//255;
    
    final static int cmtslashstar = 257;
    final static int cmtdoublehyphen = 258;
    final static int lexspace = 259;
    final static int lexnewline = 260;
    final static int fconst  = 261;
    final static int sconst = 262;
    final static int iconst = 263;
    final static int ident = 264;
    final static int op = 265;
    final static int cmpop = 266;
    final static int bind_v = 267;
    final static int assign_sign = 268;
    final static int double_dot = 269;
    final static int label_begin = 270;
    final static int label_end = 271;
    final static int substitution_v  = 272;
    final static int filepath_sign = TBaseType.filepath_sign;
    final static int sqlpluscmd = 273;
    final static int atversion = TBaseType.atversion; //databricks
    final static int error = 274;
    final static int variable = 275;
    final static int mslabel = 276;
    public final static int bconst = TBaseType.bconst; //postgresql
    final static int leftjoin_op = 277;
    final static int odbc_esc_prefix = 277;
    final static int rightjoin_op = 278;
    final static int odbc_esc_terminator = 278;
    final static int db2label = 279;
    public final static int xconst = TBaseType.xconst; //postgresql
    final static int ref_arrow = 280;
    final static int rw_scriptoptions = 281;
    public final static int UNICODE_ENCODE_ID = 281;
    final static int mysqllabel = 282;
    final static int NAMED_PARAMETER_SIGN = 282; //oracle,db2,snowflake CALL update_order (5000, NEW_STATUS => 'Shipped')
    final static int QUOTED_IDENT = 282;//used in mdx
    final static int BTEQCMD = 282;
    final static int concatenationop = 283;
    final static int pipe_greater = TBaseType.pipe_greater; // StarRocks pipe operator |>
    final static int rw_not_deferrable = 284;
    final static int rw_for1 = 285;
    final static int stmt_delimiter = 286;
    final static int AMP_QUOTED_ID = 285; //used in mdx
    final static int AMP_UNQUOTED_ID = 286; //used in mdx
    final static int m_clause = 287;
    final static int MySQL_CHARSET_NAME = 287;
    final static int typecast = TBaseType.typecast;//postgresql
    final static int k_clause = 288;
    final static int slash_dot = 288;
    final static int outer_join = 289;

    final static int not_equal = 290;

    final static int param = TBaseType.param;
    final static int mysql_null = TBaseType.rrw_mysql_null;

    final static int rw_locktable = 296;
    final static int rw_foreign2 = 297;
    final static int rw_constraint2 = 298;
    final static int rw_primary2 = 299;
    final static int rw_unique2 = 300;
    final static int     NEXT_PARAM = TBaseType.NEXT_PARAM;
    final static int     POSITIONAL_PARAM = TBaseType.POSITIONAL_PARAM;
    final static int     NAMED_PARAM = TBaseType.NAMED_PARAM;

    final static int castoperator = TBaseType.castoperator;
    final static int twocolons = TBaseType.twocolons;
    final static int compoundAssignmentOperator = TBaseType.compoundAssignmentOperator;
    final static int postgresql_function_delimiter = TBaseType.rrw_postgresql_function_delimiter;
    final static int greenplum_function_delimiter = TBaseType.rrw_greenplum_function_delimiter;

    final static int redshift_function_delimiter = TBaseType.rrw_redshift_function_delimiter;
    final static int snowflake_function_delimiter = TBaseType.rrw_snowflake_function_delimiter;



    int[] yypos;// = new int[max_rules + 1];      // 1 based in delphi, Position 0 was not used here
    int[] yystack;// = new int[max_matches + 1];  // 1 based in delphi, Position 0 was not used here
  //  ArrayList yystack;

    //String keywordvaluefile,keywordfile,yyk_file,yym_file,yykl_file;
    //String yykh_file,yyml_file,yymh_file,yytl_file,yyth_file,yytint_file,yyt_file;

    EDbVendor dbvendor;
    TSourceToken prevToken = null;

    public void setSqlCharset(String sqlCharset) {
        this.sqlCharset = sqlCharset;
    }

    public String getSqlCharset() {
        return sqlCharset;
    }

    private String sqlCharset = null;
    
    /**
     * Check if token code represents a single character operator
     */
    protected boolean isSingleCharOperator(int tokenCode) {
        return tokenCode == '(' || tokenCode == ')' || 
               tokenCode == '[' || tokenCode == ']' ||
               tokenCode == '{' || tokenCode == '}' ||
               tokenCode == ',' || tokenCode == ';' ||
               tokenCode == '.' || tokenCode == ':' ||
               tokenCode == '+' || tokenCode == '-' ||
               tokenCode == '*' || tokenCode == '/' ||
               tokenCode == '%' || tokenCode == '=' ||
               tokenCode == '<' || tokenCode == '>' ||
               tokenCode == '!' || tokenCode == '&' ||
               tokenCode == '|' || tokenCode == '^' ||
               tokenCode == '~' || tokenCode == '?';
    }
    
    /**
     * Check if token code represents a keyword
     */
    protected boolean isKeyword(int tokenCode) {
        // Check if it's in the reserved word range
        return tokenCode >= TBaseType.rrw_select && tokenCode < TBaseType.rrw_abort;
    }

    public TCustomLexer(){
       //this.yyinput = pbuf;
       yytextbufsize = intial_bufsize - 1;
       yytextbuf = new char[intial_bufsize];
       checkyytextbuf(yytextbufsize);

       bufsize = intial_bufsize - 1;
       buf = new char[intial_bufsize];
       checkbuf(bufsize);

       //literalbuf = new StringBuffer();
        literalbuf = new StringBuilder();
        //keywordList = new TreeMap();
        delimiterchar = ';';
        tmpDelimiter = "";

        xcdepth = 0;
        nchars = 0;
        isqmarktoident = true;

       yylvalstr = "";
        yysstate = 0;
        yylstate = 0;
        yymatches = 0;
        yysleng = 0;
       bufptr = 0;
       yylineno = 0;
       yycolno = 0;
       offset = -1;
       yylineno_p = 1;
       yycolno_p = 1;
       offset_p = 0;

       yypos = new int[max_rules + 1];
       max_matches = TBaseType.LEXER_INIT_MAX_MATCHES;
       yystack = new int[max_matches + 1];

        prevToken = null;
    }

    /*
     * this function is not used. 
    private void getkeywordvaluefromfile(){
        int i;
        keywordValueList.clear();
        for(i=0; i<keywordlist.length; i++){
           // System.out.println(keywordlist[i]);
            String[] ss = keywordlist[i].split("[=]");
            keywordValueList.put(ss[0].toUpperCase(Locale.ROOT),ss[1]);
        }
    }
     */

public  int iskeyword(String str){
    return -1;
}

public boolean isAtBeginOfLine(){
    return (yyretval == lexnewline || yyretval == 0);
}

//public boolean canBeColumnName(int tokencode){
//    return false;
//}


public String getStringByCode(int tokenCode){
    return null;
}

    public  int getkeywordvalue(String keyword){
        return 0;
    }


    /**
     * @deprecated , please use keywordChecker.isKeyword() instead.
     *
     * because there are so many non-reserved keywords in some databases, it's not suitable to put those
     * non-reserved keywords in lexer and parser.
     *
     * @param keyword
     * @param keywordValueList
     * @param keywordTypeList
     * @return
     */
    public static EKeywordType getKeywordType(String keyword, HashMap<String, Integer> keywordValueList,HashMap<Integer, Integer> keywordTypeList){
        EKeywordType ret = EKeywordType.NOT_A_KEYWORD;
        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
        if( s == null) return ret;

        Integer i = keywordTypeList.get(s);
        if (i == 1) return EKeywordType.RESERVED_WORD;
        else if (i == 2) return EKeywordType.NON_RESERVED_KEYWORD;
        else return  ret;
    }

    /**
     * 如果是ascii 字符，直接返回，如果是unicode 字符，需要进行转换。否则 String.charAt() 返回的unicode字符不是我们想要的字符，
     * 例如中文的括号，我们实际需要的ascii的括号
     *
     * @param pYylvalstr
     * @param index
     * @return
     */
   char lexer_charAt(String pYylvalstr,int index){
        char ret = pYylvalstr.charAt(index);
        if (ret > 255){
            // this is a unicode code
            if ((ret == 0xFF08)){
                // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=65280&number=128
                // Unicode code point for FULLWIDTH LEFT PARENTHESIS (, 0xFF08
                //System.out.println(c);
                ret = '(';
            }
            if ( (ret == 0xFF09)){
                // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=65280&number=128
                // Unicode code point for FULLWIDTH RIGHT PARENTHESIS ), 0xFF09
                // System.out.println(c);
                ret = ')';
            }
            if (ret == 0xFF0C && dbvendor == EDbVendor.dbvoracle) {
                // totablechar already selects the comma rule. Return its token code,
                // while retaining the original character in literals, comments and SQL text.
                ret = ',';
            }
        }
        return ret;
   }
   void totablechar(){
       //System.out.println("char:"+yyactchar+" ,hex:"+String.format("%04x", (int) yyactchar));
       //System.out.println(String.format("0x%08X", (int)yyactchar)+", "+(char)yyactchar);

    if (((int) yyactchar == 0) && !endOfInput) {
        yytablechar = (char)255;
        return;
    }

     if ((int)(yyactchar) < 228){ // 228 is ä in unicode
       yytablechar = yyactchar;
         if ((((int)(yyactchar) == 160)&&(utf8NoBreakSpaceReady))||(yyactchar == 0xA0)){
             yytablechar = (char)32;
         }
       utf8NoBreakSpaceReady = false;
//         if (yyactchar == 0x27){
//             insideSingleQuoteStr = !insideSingleQuoteStr;
//          }
     }else{
         yytablechar = (char)'a';//(char)255;

         if ((int)(yyactchar) == 914) { // c2 a0, utf-8 NO-BREAK SPACE
             utf8NoBreakSpaceReady = true;
             yytablechar = (char) 32;
         }else if ((yyactchar == 0x2018)||(yyactchar == 0x2019)){
             if (stringLiteralStartWithUnicodeSingleQuote){
                 // WHERE Name LIKE ‘Acme%’
                 // 如上，如果string literal 以unicode quote 开始，则不管当前是否在string literal中，新碰到的unicode quote都看成是string literal的结尾符，
                 yytablechar = 0x27; // treat  Unicode Character 'LEFT SINGLE QUOTATION MARK' as the ascii char ', but don't change it
             }else {
                 if (insideSingleQuoteStr){
                     // don't change the unicode quote char
                 }else {
                     yytablechar = 0x27; // treat  Unicode Character 'LEFT SINGLE QUOTATION MARK' as the ascii char ', but don't change it
                 }
             }

         }else if ((yyactchar == 0x200B)||(yyactchar == 0x3000)||(yyactchar >= 0x2000 && yyactchar <= 0x200A)){
             // Unicode code point 0x200B: treat  Unicode Character ZERO WIDTH SPACE as the ascii char space, but don't change it
             // Unicode code point 0x3000: treat  Unicode Character IDEOGRAPHIC SPACE (UTF-8: e3 80 80) as the ascii char space, but don't change it
             // Unicode code points 0x2000-0x200A: General Punctuation space characters (EN QUAD, EM QUAD, EN SPACE, EM SPACE, THREE-PER-EM SPACE, etc.)
             yytablechar = 0x20;
         }else if (yyactchar == 0xFF08){
             yytablechar = '('; // treat  Unicode code point for FULLWIDTH LEFT PARENTHESIS  as the ascii char (, but don't change it
         }else if (yyactchar == 0xFF09){
             yytablechar = ')'; // treat  Unicode code point for FULLWIDTH RIGHT PARENTHESIS  as the ascii char ), but don't change it
         }else if (yyactchar == 0xFF0C){
             yytablechar = ','; // treat  Unicode code point for FULLWIDTH COMMA  as the ascii char comma, but don't change it
         }else {
             utf8NoBreakSpaceReady = false;
         }
     }
   }

    String getyytext(){
      return new String(yytextbuf,0,yytextlen);
    }


    void checkyytextbuf(int size){
       while ( size >= yytextbufsize){
          yytextbufsize = yytextbufsize * 2 > intial_bufsize ? yytextbufsize * 2: intial_bufsize;
          char[] tmp = new char[yytextbufsize];
           System.arraycopy(yytextbuf,0,tmp,0, yytextbuf.length);
           yytextbuf = tmp;
       }
    }
    
    void checkbuf(int size){
       // System.out.println("while begin2"+" size:"+size+" bufsize:"+bufsize);
       while ( size >= bufsize){
          bufsize = bufsize * 2 > intial_bufsize ? bufsize * 2: intial_bufsize;
          char[] tmp = new char[bufsize];
           System.arraycopy(buf,0,tmp,0, buf.length);
           buf = tmp;
       }
       // System.out.println("while end2");
    }

    boolean eof(BufferedReader pbuf){
        try{
        return !pbuf.ready();
        }catch(IOException e){
          return true;
        }
    }

    void yynew(){
        if (yylastchar != (char)0){
          if(yylastchar == lf){
            yylstate = 1;
          }else{
              yylstate = 0;
          }
        }

        yystate = yysstate + yylstate;
        checkyytextbuf(0);
        yytextlen = 0;
        yymatches = 0;
        yydone = false;
    }

    void yyscan(){
        yyactchar = get_char();
        checkyytextbuf(yytextlen + 1);
        yytextlen++;
        yytextbuf[yytextlen - 1] = yyactchar;
    }

    void yymark(int n){
        if (n > max_rules ){
           System.out.println("n > max_rules ");
        }
        yypos[n] = yytextlen;
    }

    void yymatch(int n){
        yymatches++;
        if(yymatches > max_matches){

            int new_yystack[] = new int[max_matches*2+1];
            System.arraycopy(yystack, 0, new_yystack, 0, max_matches);
            yystack = new_yystack;
            max_matches = max_matches * 2;

           // this is valid in JDK 1.6, proguard will report warning and stop
           // yystack = Arrays.copyOf(yystack,max_matches+1);

        }
        yystack [yymatches] = n;
    }

    int yyfind(){
        //return -1 mean not found
        int ret = -1;

        yyreject = false;
        
        while (( yymatches > 0 ) && ( yypos[yystack[yymatches]] == 0 )) {
           yymatches-- ;
        }
        

        if (yymatches > 0){
          yysleng = yytextlen;
          ret = yystack[yymatches];
          yyless( yypos[ret] );
          yypos[ret] = 0;
          if (yytextlen >0){
            yylastchar = yytextbuf [yytextlen-1];
          }else{
            yylastchar = (char)0;
          }
        }else{
          yyless( 0 );
          yylastchar = (char)0;
        }

       return ret;
    }

    boolean yydefault(){
        boolean ret;

        yyreject = false;
        yyactchar = get_char();
        if (yyactchar != (char)0){
          //put_char( yyactchar );
          ret = true;
        }else{
          yylstate = 1;
          ret = false;
        }
        yylastchar = yyactchar;
        return ret;
    }
    void yyless(int n){
        for(int i= yytextlen; i> n; i--){
            unget_char(yytextbuf[i - 1]);
        }
        checkyytextbuf(n);
        yytextlen = n;
    }
    void returni(int n){
        yyretval = n;
        yydone = true;
    }
    void returnc(char c){
        yyretval = (int)c;
        yydone = true;
    }
    void yyclear(){
        // the generated lexer clears its state when the input ends: an end inside the literal startlit() opened is
        // kept for openLiteralAtEnd (Mantis 4842)
        literalOpenAtEnd = literalPending && (yysstate == xd) && (literalState == xd) && (literalbuf != null)
                && (literalbuf.length() > 0);
        literalPending = false;
        if (literalOpenAtEnd) {
            // where the literal starts: right after the last token returned
            literalLineNo = yylineno_p;
            literalColumnNo = yycolno_p;
            literalOffset = offset_p;
        }
        bufptr = 0;
        yysstate = 0;
        yylstate = 1;
        yylastchar = (char)0;
        yytextlen = 0;
        yylineno = 0;
        yycolno = 0;
        offset = -1;
       // yystext := '';

        yylineno_p = 1;
        yycolno_p = 1;
        offset_p = 0;
        literalState = 0;

    }

    
    boolean yywrap(){
        return true;
    }
    int getyysstate(){
        return yysstate;
    }
    void start(int pstate){
        yysstate = pstate;
        if (pstate == xq){
            insideSingleQuoteStr = true;
            if ((yylvalstr.charAt(0) == 0x2018)||(yylvalstr.charAt(0) == 0x2019)){
                stringLiteralStartWithUnicodeSingleQuote = true;
            }else{
                stringLiteralStartWithUnicodeSingleQuote = false;
            }
        }else{
            insideSingleQuoteStr = false;
        }
    }


    void unget_char(char pchar){
        // get_char() uses NUL as its end-of-input sentinel and does not
        // advance yycolno/offset when returning it.  Pushing that sentinel
        // back used to decrement both counters as if a real source character
        // had been consumed.  A following yyless() split at EOF could then
        // assign the next real token the preceding token's source span (for
        // example, DB2 "k=?" gave '?' the '=' offset).  Only ignore the EOF
        // sentinel: a real NUL input character does advance the position and
        // must still be restored.
        if (pchar == (char)0 && endOfInput) {
            endOfInput = false;
            return;
        }
        if(bufptr == max_chars)
        {
            System.out.println("input buffer overflow");
        }
      //  if (bufptr > 0) {
        bufptr++;
        yycolno--;
        offset--;
        checkbuf(bufptr+1);
        buf[bufptr] = pchar;
      //  }

    }

    public void reset(){
        insideSingleQuoteStr = false;
        nestedLessThan = 0;
        literalState = 0;
        literalPending = false;
        literalOpenAtEnd = false;
    }

    public boolean insideSingleQuoteStr = false;
    public boolean stringLiteralStartWithUnicodeSingleQuote = false;


    // Previous implementation of readln, 2025-05-04
    // char[] readln()  throws IOException {
    //     int c;
    //     char[] buffer = new char[80];
    //     int bufferSize = 0;

    //     while ((c = yyinput.read()) != -1) {
    //         if (bufferSize >= buffer.length) {
    //             char[] newBuffer = new char[buffer.length * 2];
    //             System.arraycopy(buffer, 0, newBuffer, 0, buffer.length);
    //             buffer = newBuffer;
    //         }

    //         buffer[bufferSize++] = (char)c;

    //         if (c == '\n' || c == '\r') {
    //             break;
    //         }
    //     }

    //     if (bufferSize > 0 && buffer[bufferSize - 1] == '\r') {
    //         yyinput.mark(1);
    //         c = yyinput.read();
    //         if (c == '\n') {
    //             if (bufferSize >= buffer.length) {
    //                 char[] newBuffer = new char[buffer.length + 1];
    //                 System.arraycopy(buffer, 0, newBuffer, 0, buffer.length);
    //                 buffer = newBuffer;
    //             }
    //             buffer[bufferSize++] = '\n';
    //         } else {
    //             yyinput.reset();
    //         }
    //     }

    //     if (bufferSize == 0) {
    //         return null;
    //     }else{
    //         char[] result = new char[bufferSize];
    //         System.arraycopy(buffer, 0, result, 0, bufferSize);
    //         return result;
    //     }
    // }

/**
 * High-performance line reader with optimal buffer management.
 * Reuses lineReadBuffer across calls to reduce per-line allocations.
 * @return char array containing the line including line ending, or null if end of stream
 */
char[] readln() throws IOException {
    if (yyinput == null) return null;

    int position = 0;
    int c;

    // Read characters until line ending or EOF
    while ((c = yyinput.read()) != -1) {
        // Expand buffer if needed (expanded buffer stays for reuse)
        if (position >= lineReadBuffer.length) {
            char[] newBuffer = new char[lineReadBuffer.length * 2];
            System.arraycopy(lineReadBuffer, 0, newBuffer, 0, lineReadBuffer.length);
            lineReadBuffer = newBuffer;
        }

        // Store character
        lineReadBuffer[position++] = (char)c;

        // Check for line endings
        if (c == '\n') {
            break; // LF - end of line
        } else if (c == '\r') {
            // Need to check for CR+LF sequence
            yyinput.mark(1);
            c = yyinput.read();

            if (c == '\n') {
                // CR+LF sequence - include LF in result
                if (position >= lineReadBuffer.length) {
                    char[] newBuffer = new char[lineReadBuffer.length + 1];
                    System.arraycopy(lineReadBuffer, 0, newBuffer, 0, lineReadBuffer.length);
                    lineReadBuffer = newBuffer;
                }
                lineReadBuffer[position++] = '\n';
            } else {
                // CR only - reset stream to keep the character after CR
                yyinput.reset();
            }
            break;
        }
    }

    // Return null if no characters were read (end of stream). yylineLen keeps the
    // length of the last line, which yyline still holds (see currentLineText()).
    if (position == 0) {
        return null;
    }

    // Return lineReadBuffer directly, avoiding per-line array allocation.
    // yylineLen holds the actual content length (replaces yyline.length semantic).
    yylineLen = position;
    return lineReadBuffer;
}

    /**
     * Text of the line being scanned: only the characters of the last line read,
     * never the tail that a longer earlier line left in the reused buffer. At end
     * of input it is still the last line, because the action of the final token
     * runs after the reader is exhausted. Empty before the first line is read.
     */
    String currentLineText(){
        return (yyline == null) ? "" : new String(yyline, 0, yylineLen);
    }

    char get_char(){

        char ret ;
         boolean readlineok = true;

        if ((bufptr == 0) && !eof(yyinput) )
        {
            try{
               endOfInput = false;
               char[] line = readln();//yyinput.readLine();
              // System.out.println("readln: "+yyline);
                if (line == null){
                  // End of input. Keep yyline on the last line read: the action of the
                  // token that ends the input runs after this call and may still read it
                  // (the SQL Server "--" rule does, Mantis 4951).
                  readlineok = false;
                }  else{
                    yyline = line;
                    yylineno++;
                    yycolno = 0;
                    // Use yylineLen instead of yyline.length since lineReadBuffer is reused
                    bufptr = yylineLen;
                    checkbuf(bufptr+1);
                    for(int k=1;k<=bufptr;k++){
                        buf[k] = yyline[bufptr - k];
                    }
                }
            }catch(IOException e){
              readlineok = false;
            }
        }

        if (! readlineok){
          endOfInput = true;
          return (char)0;
        }

       if (bufptr > 0){
         bufptr--;
         yycolno++;
         offset++;

         return buf[bufptr+1];
         //return yyline.charAt(yyline.length()  - (bufptr + 1));
       }else{
       // bufptr--;
           endOfInput = true;
        return  (char)0;
       }

    }

    void startlit(){
        literalbuf.setLength(0);
        literallen = 0;
        literalalloc = 0;
        literalState = yysstate;
        literalPending = true;
    }

    /** The start state the literal in {@link #literalbuf} was opened in, 0 when none (Mantis 4842). */
    private int literalState;
    /** Whether no token was returned since {@link #startlit()} opened that literal (Mantis 4842). */
    private boolean literalPending;
    /** Whether the input ended inside the literal {@link #startlit()} opened, not yet returned (Mantis 4842). */
    private boolean literalOpenAtEnd;
    /** Where that literal starts. */
    private long literalLineNo, literalColumnNo, literalOffset;

    /**
     * The double-quoted identifier the input ends in, as one more token (Mantis 4842). The lexer reaches the end inside
     * the identifier's start state and returns no token for it, so its opening quote and everything after it were
     * lost: {@code SELECT 1 id";} parsed as {@code SELECT 1 id}, and {@code SELECT 1 id"; SELECT 2;} lost its second
     * statement. The token's code, the double quote character, is in no grammar, so the parse fails at it. An input
     * ending inside a string or a comment still ends there: corpora and customer scripts end that way, some because a
     * lexer misreads a string (U&amp;'', Vertica's $$), and need a fix of their own first.
     *
     * @return the token's code, or 0 when the input ends outside a literal
     */
    private int openLiteralAtEnd(TSourceToken psourcetoken) {
        if (!literalOpenAtEnd) {
            return 0;
        }
        literalOpenAtEnd = false;
        psourcetoken.tokencode = '"';
        psourcetoken.tokentype = ETokenType.tttokenlizererrortoken;
        psourcetoken.setAstext(literalbuf.toString());
        psourcetoken.lineNo = literalLineNo;
        psourcetoken.columnNo = literalColumnNo;
        psourcetoken.offset = literalOffset;
        return psourcetoken.tokencode;
    }

    void addlit(String ytext, int yleng){
        literallen = literallen + yleng;
        literalbuf.append(ytext,0,yleng);
    }

    void addlitchar(char ychar){
        literallen++;
        literalbuf.append(ychar);
    }

    String litbufdup(){
        return literalbuf.toString();//.intern();
    }

    boolean isopchar(char ch){
        switch (ch) {
            case '~':
            case '!':
            case '@':
            case '#':
            case '^':
            case '&':
            case '|':
            case '`':
            case '?':
            case '$':
            case '%':
                return true;
            default:
                return false;
        }
    }

    boolean isselfchar(char ch){
        switch (ch) {
            case ',':
            case '(':
            case ')':
            case '[':
            case ']':
            case '.':
            case ';':
            case '$':
            case ':':
            case '+':
            case '-':
            case '*':
            case '/':
            case '%':
            case '^':
            case '<':
            case '>':
            case '=':
            case '!':
            case '{':
            case '}':
                return true;
            default:
                return false;
        }
    }

    boolean charinarray(char c, char[] a){
        int len = a.length;
        for (int i = 0; i < len; i++) {
            if (a[i] == c)
                return true;
        }
        return false;
    }

    void setlengthofliteralbuf(int plen){
      literalbuf.setLength(plen);
    }

    void yyaction(int yyruleno){
    }

    int yylex(){
        return 0;
    }


    public int yylexwrap(TSourceToken psourcetoken) {
        // Get token code and handle EOF
        if ((psourcetoken.tokencode = yylex()) == 0) return openLiteralAtEnd(psourcetoken);
        // a lexer may return the literal and stay in its start state (Snowflake ""name""): it is no longer open
        literalPending = false;

        // Store token text - use shared strings for single ASCII chars to reduce allocations
        if (yylvalstr == null) {
            if (yytextlen == 1 && yytextbuf[0] < 128) {
                yylvalstr = SINGLE_CHAR_STRINGS[yytextbuf[0]];
            } else {
                yylvalstr = new String(yytextbuf, 0, yytextlen);
            }
        }
        psourcetoken.setAstext(yylvalstr);
    
        // Record token position information
        psourcetoken.lineNo = yylineno_p;
        psourcetoken.columnNo = yycolno_p;
        psourcetoken.offset = offset_p;
        yylineno_p = yylineno;
        yycolno_p = yycolno + 1;
        offset_p = offset + 1;
        
        // Track token in token table for analysis
        setTokenTableValue(psourcetoken);
    
        // Handle token types based on token code
        switch (psourcetoken.tokencode) {
            case cmtdoublehyphen:
                psourcetoken.tokentype = ETokenType.ttsimplecomment;
                if (dbvendor == EDbVendor.dbvmdx && psourcetoken.toString().startsWith("/")) {
                    psourcetoken.tokentype = ETokenType.ttCPPComment;
                }
                break;
                
            case cmtslashstar:
                psourcetoken.tokentype = ETokenType.ttbracketedcomment;
                break;
                
            case lexspace:
                psourcetoken.tokentype = ETokenType.ttwhitespace;
                break;
                
            case lexnewline:
                psourcetoken.tokentype = ETokenType.ttreturn;
                break;
                
            case bind_v:
                psourcetoken.tokentype = ETokenType.ttbindvar;
                if (dbvendor == EDbVendor.dbvoracle) {
                    psourcetoken.setAstext(psourcetoken.getAstext().replace(TBaseType.newline, ""));
                }
                break;
                
            case stmt_delimiter:
                psourcetoken.tokentype = ETokenType.ttstmt_delimiter;
                psourcetoken.tokencode = cmtslashstar;
                break;
                
            case concatenationop:
                psourcetoken.tokentype = ETokenType.ttconcatenationop;
                break;
                
            case variable:
                psourcetoken.tokentype = ETokenType.ttsqlvar;
                break;
                
            case fconst:
            case iconst:
                psourcetoken.tokentype = ETokenType.ttnumber;
                break;
                
            case sconst:
                psourcetoken.tokentype = ETokenType.ttsqstring;
                psourcetoken.dolqstart = dolqstart;
                dolqstart = "";
                break;
                
            case ident:
            case QUOTED_IDENT:
                handleIdentifierToken(psourcetoken);
                break;
                
            case cmpop:
                handleComparisonOperator(psourcetoken);
                break;
                
            case op:
                handleOperatorToken(psourcetoken);
                break;
                
            default:
                handleDefaultToken(psourcetoken);
                break;
        }
    
        prevToken = psourcetoken;
        return psourcetoken.tokencode;
    }
    
    // Helper methods to better organize the complex token handling logic
    private void handleIdentifierToken(TSourceToken psourcetoken) {
        psourcetoken.tokentype = ETokenType.ttidentifier;
        String tokenText = psourcetoken.toString().trim();
        
        if (tokenText.startsWith("\"")) {
            psourcetoken.tokentype = ETokenType.ttdqstring;
        } else if (tokenText.startsWith("[")) {
            if (dbvendor == EDbVendor.dbvmssql || dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq) {
                psourcetoken.tokentype = ETokenType.ttdbstring;
            }
        } else if (tokenText.startsWith("{")) {
            if (dbvendor == EDbVendor.dbvmssql || dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq) {
                psourcetoken.tokentype = ETokenType.ttbrstring;
                if (tokenText.toLowerCase(Locale.ROOT).startsWith("{escape")) {
                    psourcetoken.tokencode = TBaseType.rrw_sqlserver_odbc_escape;
                }
            }
        } else if (tokenText.startsWith("&")) {
            if (dbvendor == EDbVendor.dbvmdx) {
                if (psourcetoken.tokencode == QUOTED_IDENT) {
                    psourcetoken.tokencode = AMP_QUOTED_ID;
                } else if (psourcetoken.tokencode == ident) {
                    psourcetoken.tokencode = AMP_UNQUOTED_ID;
                }
            }
        } else if (tokenText.startsWith(".")) {
            if (dbvendor == EDbVendor.dbvteradata) {
                psourcetoken.tokentype = ETokenType.ttBTEQCmd;
            }
        }
    }
    
    private void handleComparisonOperator(TSourceToken psourcetoken) {
        psourcetoken.tokentype = ETokenType.ttmulticharoperator;
        String token = yylvalstr;

        // Oracle 26c vector distance operators (3-char, Oracle only)
        if (dbvendor == EDbVendor.dbvoracle) {
            if (token.equals("<=>")) {
                psourcetoken.tokencode = TBaseType.vector_cosine_distance;
                return;
            } else if (token.equals("<->")) {
                psourcetoken.tokencode = TBaseType.vector_euclidean_distance;
                return;
            } else if (token.equals("<#>")) {
                psourcetoken.tokencode = TBaseType.vector_dot_product;
                return;
            }
        }

        if ((token.startsWith("!") && token.endsWith("=")) ||
            (token.startsWith("^") && token.endsWith("=")) ||
            (token.startsWith("~") && token.endsWith("=")) ||
            (token.startsWith("<") && token.endsWith(">"))) {

            psourcetoken.tokencode = TBaseType.not_equal;

            // Handle MySQL NULL-safe equal
            if (token.indexOf("=", 1) > 0 &&
                token.startsWith("<") && token.endsWith(">")) {
                psourcetoken.tokencode = (int)'=';
            }
        } else if (token.startsWith(">") && token.endsWith("=")) {
            psourcetoken.tokencode = TBaseType.great_equal;
        } else if (token.startsWith("<") && token.endsWith("=")) {
            psourcetoken.tokencode = TBaseType.less_equal;
        } else if ((token.startsWith("!") && token.endsWith("<")) ||
                   (token.startsWith("^") && token.endsWith("<"))) {
            psourcetoken.tokencode = TBaseType.not_less;
        } else if ((token.startsWith("!") && token.endsWith(">")) ||
                   (token.startsWith("^") && token.endsWith(">"))) {
            psourcetoken.tokencode = TBaseType.not_great;
        } else if (token.length() == 2 && token.charAt(0) == ':' && token.charAt(1) == '=') {
            psourcetoken.tokencode = assign_sign;
        }
    }
    
    private void handleOperatorToken(TSourceToken psourcetoken) {
        psourcetoken.tokentype = ETokenType.ttmulticharoperator;
        String token = yylvalstr;
        int tokenLength = token.length();
        char firstChar = tokenLength > 0 ? token.charAt(0) : '\0';
        char secondChar = tokenLength > 1 ? token.charAt(1) : '\0';
        
        // Handle question mark specially
        if (token.equals("?") && isqmarktoident) {
            handleQuestionMark(psourcetoken);
            return;
        }
        
        // Handle special two-character operators
        if (tokenLength == 2) {
            if (handleTwoCharOperator(psourcetoken, firstChar, secondChar)) {
                return;
            }
        }
        
        // Handle special three-character operators
        if (tokenLength == 3) {
            if (handleThreeCharOperator(psourcetoken, firstChar, secondChar, token.charAt(2))) {
                return;
            }
        }
        
        // Handle comparison operators
        if (handleComparisonOp(psourcetoken, token)) {
            return;
        }
        
        // Handle single character operators
        if (tokenLength == 1) {
            handleSingleCharOperator(psourcetoken, firstChar);
        }
    }
    
    private boolean handleTwoCharOperator(TSourceToken psourcetoken, char firstChar, char secondChar) {
        switch (firstChar) {
            case '<':
                if (secondChar == '<') {
                    return handleLeftShiftOperator(psourcetoken);
                } else if (secondChar == '@') {
                    psourcetoken.tokencode = TBaseType.JSON_RIGHT_CONTAIN;
                    return true;
                }
                break;
                
            case '>':
                if (secondChar == '>') {
                    return handleRightShiftOperator(psourcetoken);
                }
                break;
                
            case '=':
                if (secondChar == '>') {
                    if (dbvendor == EDbVendor.dbvodbc) {
                        psourcetoken.tokencode = TBaseType.great_equal;
                    } else if (dbvendor == EDbVendor.dbvpostgresql || dbvendor == EDbVendor.dbvgaussdb || dbvendor == EDbVendor.dbvedb) {
                        psourcetoken.tokencode = TBaseType.assign_sign;
                    } else {
                        psourcetoken.tokencode = NAMED_PARAMETER_SIGN;
                    }
                    return true;
                } else if (secondChar == '*') {
                    if (dbvendor == EDbVendor.dbvmssql || dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq) {
                        psourcetoken.tokencode = rightjoin_op;
                    }
                    return true;
                } else if (secondChar == '<') {
                    if (dbvendor == EDbVendor.dbvodbc) {
                        psourcetoken.tokencode = TBaseType.less_equal;
                    }
                    return true;
                } else if (secondChar == '=') {
                    if (dbvendor == EDbVendor.dbvsparksql || dbvendor == EDbVendor.dbvclickhouse) {
                        psourcetoken.tokencode = '=';
                    }
                    return true;
                }
                break;
                
            case '-':
                if (secondChar == '>') {
                    if (dbvendor == EDbVendor.dbvpostgresql || dbvendor == EDbVendor.dbvgaussdb || dbvendor == EDbVendor.dbvedb 
                        || dbvendor == EDbVendor.dbvgreenplum || dbvendor == EDbVendor.dbvmysql) {
                        psourcetoken.tokencode = TBaseType.JSON_GET_OBJECT;
                    } else {
                        psourcetoken.tokencode = ref_arrow;
                    }
                    return true;
                } else if (secondChar == '=') {
                    if (dbvendor == EDbVendor.dbvmssql || dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq) {
                        psourcetoken.tokencode = compoundAssignmentOperator;
                    }
                    return true;
                }
                break;
                
            case '.':
                if (secondChar == '.') {
                    if (dbvendor == EDbVendor.dbvdb2 || dbvendor == EDbVendor.dbvoracle 
                        || dbvendor == EDbVendor.dbvmysql || dbvendor == EDbVendor.dbvhana) {
                        psourcetoken.tokencode = double_dot;
                    }
                    return true;
                }
                break;
                
            case '*':
                if (secondChar == '=') {
                    if (dbvendor == EDbVendor.dbvmssql || dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq) {
                        psourcetoken.tokencode = leftjoin_op;
                    }
                    return true;
                } else if (secondChar == '*') {
                    if (dbvendor == EDbVendor.dbvteradata || dbvendor == EDbVendor.dbvnetezza) {
                        psourcetoken.tokencode = TBaseType.exponentiate;
                    }
                    return true;
                }
                break;
                
            case '|':
                if (secondChar == '|') {
                    if (dbvendor == EDbVendor.dbvmysql) {
                        psourcetoken.tokencode = TBaseType.logical_or;
                    } else if (isStringConcatVendor(dbvendor)) {
                        psourcetoken.tokencode = TBaseType.concatenationop;
                    }
                    return true;
                } else if (secondChar == '>') {
                    if (dbvendor == EDbVendor.dbvsparksql) {
                        psourcetoken.tokencode = TBaseType.sparksql_pipe_arrow;
                    } else {
                        psourcetoken.tokencode = TBaseType.pipe_greater;
                    }
                    return true;
                } else if (secondChar == '/') {
                    if (dbvendor == EDbVendor.dbvredshift) {
                        psourcetoken.tokencode = TBaseType.square_root;
                    }
                    return true;
                }
                break;
                
            case '&':
                if (secondChar == '&') {
                    if (dbvendor == EDbVendor.dbvmysql) {
                        psourcetoken.tokencode = TBaseType.logical_and;
                    }
                    return true;
                }
                break;
                
            case '?':
                if (secondChar == '|') {
                    psourcetoken.tokencode = TBaseType.JSON_ANY_EXIST;
                    return true;
                } else if (secondChar == '&') {
                    psourcetoken.tokencode = TBaseType.JSON_ALL_EXIST;
                    return true;
                }
                break;
                
            case '@':
                if (secondChar == '>') {
                    psourcetoken.tokencode = TBaseType.JSON_LEFT_CONTAIN;
                    return true;
                }
                break;
                
            case '#':
                if (secondChar == '>') {
                    psourcetoken.tokencode = TBaseType.JSON_GET_OBJECT_AT_PATH;
                    return true;
                }
                break;
                
            case ':':
                if (secondChar == '=') {
                    psourcetoken.tokencode = assign_sign;
                    return true;
                }
                break;
        }
        
        // Handle compound assignment operators
        if ((firstChar == '+' || firstChar == '-' || firstChar == '*' || 
             firstChar == '/' || firstChar == '%' || firstChar == '&' || 
             firstChar == '^' || firstChar == '|') && secondChar == '=') {
            if (dbvendor == EDbVendor.dbvmssql || dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq) {
                psourcetoken.tokencode = compoundAssignmentOperator;
                return true;
            } else if (dbvendor == EDbVendor.dbvmysql && firstChar == '^' && secondChar == '=') {
                psourcetoken.tokencode = not_equal;
                return true;
            }
        }
        
        return false;
    }
    
    private boolean handleThreeCharOperator(TSourceToken psourcetoken, char firstChar, char secondChar, char thirdChar) {
        if (firstChar == '-' && secondChar == '>' && thirdChar == '>') {
            psourcetoken.tokencode = TBaseType.JSON_GET_TEXT;
            return true;
        } else if (firstChar == '#' && secondChar == '>' && thirdChar == '>') {
            psourcetoken.tokencode = TBaseType.JSON_GET_TEXT_AT_PATH;
            return true;
        } else if (firstChar == '|' && secondChar == '|' && thirdChar == '/') {
            if (dbvendor == EDbVendor.dbvredshift) {
                psourcetoken.tokencode = TBaseType.cube_root;
                return true;
            }
        }
        return false;
    }
    
    private boolean handleComparisonOp(TSourceToken psourcetoken, String token) {
        if ((token.startsWith("!") && token.endsWith("=")) ||
            (token.startsWith("^") && token.endsWith("=")) ||
            (token.startsWith("<") && token.endsWith(">"))) {
            psourcetoken.tokencode = TBaseType.not_equal;
            return true;
        } else if (token.startsWith(">") && token.endsWith("=")) {
            psourcetoken.tokencode = TBaseType.great_equal;
            return true;
        } else if (token.startsWith("<") && token.endsWith("=")) {
            psourcetoken.tokencode = TBaseType.less_equal;
            return true;
        } else if ((token.startsWith("!") && token.endsWith("<")) ||
                   (token.startsWith("^") && token.endsWith("<"))) {
            psourcetoken.tokencode = TBaseType.not_less;
            return true;
        } else if ((token.startsWith("!") && token.endsWith(">")) ||
                   (token.startsWith("^") && token.endsWith(">"))) {
            psourcetoken.tokencode = TBaseType.not_great;
            return true;
        }
        return false;
    }
    
    private void handleSingleCharOperator(TSourceToken psourcetoken, char ch) {
        switch (ch) {
            case '~':
                if (dbvendor == EDbVendor.dbvmysql || dbvendor == EDbVendor.dbvredshift || 
                    dbvendor == EDbVendor.dbvsnowflake) {
                    psourcetoken.tokencode = (int)'~';
                }
                break;
                
            case '#':
                if (dbvendor == EDbVendor.dbvmssql) {
                    psourcetoken.tokencode = (int)'#';
                }
                break;
                
            case '&':
                if (dbvendor == EDbVendor.dbvmysql || dbvendor == EDbVendor.dbvvertica || 
                    dbvendor == EDbVendor.dbvsparksql) {
                    psourcetoken.tokencode = (int)'&';
                }
                break;
                
            case '|':
                if (dbvendor == EDbVendor.dbvmysql || dbvendor == EDbVendor.dbvvertica) {
                    psourcetoken.tokencode = (int)'|';
                }
                break;
        }
    }
    
    private void handleQuestionMark(TSourceToken psourcetoken) {
        if (dbvendor == EDbVendor.dbvpostgresql || dbvendor == EDbVendor.dbvgaussdb || dbvendor == EDbVendor.dbvedb || 
            dbvendor == EDbVendor.dbvgreenplum) {
            psourcetoken.tokencode = TBaseType.JSON_EXIST;
        } else if (dbvendor == EDbVendor.dbvodbc) {
            psourcetoken.tokencode = '?';
        } else if (dbvendor == EDbVendor.dbvsnowflake) {
            psourcetoken.tokencode = bind_v;
            psourcetoken.tokentype = ETokenType.ttquestionmark;
        } else {
            psourcetoken.tokencode = ident;
        }
    }
    
    private boolean handleLeftShiftOperator(TSourceToken psourcetoken) {
        if (dbvendor == EDbVendor.dbvoracle || dbvendor == EDbVendor.dbvmssql ||
            dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq || dbvendor == EDbVendor.dbvpostgresql ||
            dbvendor == EDbVendor.dbvgaussdb || dbvendor == EDbVendor.dbvedb || dbvendor == EDbVendor.dbvaccess ||
            dbvendor == EDbVendor.dbvgreenplum || dbvendor == EDbVendor.dbvsnowflake) {
            psourcetoken.tokencode = label_begin;
        } else if (dbvendor == EDbVendor.dbvmysql) {
            psourcetoken.tokencode = TBaseType.rrw_left_shift;
        } else if (dbvendor == EDbVendor.dbvredshift) {
            psourcetoken.tokencode = TBaseType.bitwise_shift_left;
        } else if (dbvendor == EDbVendor.dbvnetezza) {
            psourcetoken.tokencode = TBaseType.rrw_netezza_op_less_less;
        }
        return true;
    }
    
    private boolean handleRightShiftOperator(TSourceToken psourcetoken) {
        if (dbvendor == EDbVendor.dbvoracle || dbvendor == EDbVendor.dbvmssql ||
            dbvendor == EDbVendor.dbvsybase || dbvendor == EDbVendor.dbvsybasease || dbvendor == EDbVendor.dbvsqlanywhere || dbvendor == EDbVendor.dbvsybaseiq || dbvendor == EDbVendor.dbvpostgresql ||
            dbvendor == EDbVendor.dbvgaussdb || dbvendor == EDbVendor.dbvedb || dbvendor == EDbVendor.dbvgreenplum ||
            dbvendor == EDbVendor.dbvaccess || dbvendor == EDbVendor.dbvsnowflake) {
            psourcetoken.tokencode = label_end;
        } else if (dbvendor == EDbVendor.dbvmysql) {
            psourcetoken.tokencode = TBaseType.rrw_right_shift;
        } else if (dbvendor == EDbVendor.dbvredshift) {
            psourcetoken.tokencode = TBaseType.bitwise_shift_right;
        } else if (dbvendor == EDbVendor.dbvnetezza) {
            psourcetoken.tokencode = TBaseType.rrw_netezza_op_great_great;
        } else if (dbvendor == EDbVendor.dbvdoris) {
            psourcetoken.tokencode = TBaseType.rrw_doris_right_shift;
        } else if (dbvendor == EDbVendor.dbvstarrocks) {
            psourcetoken.tokencode = TBaseType.rrw_starrocks_right_shift;
        }
        return true;
    }
    
    private boolean isStringConcatVendor(EDbVendor vendor) {
        return vendor == EDbVendor.dbvdb2 || vendor == EDbVendor.dbvnetezza || 
               vendor == EDbVendor.dbvpostgresql || vendor == EDbVendor.dbvgaussdb || vendor == EDbVendor.dbvedb ||
               vendor == EDbVendor.dbvredshift || vendor == EDbVendor.dbvgreenplum || 
               vendor == EDbVendor.dbvbigquery || vendor == EDbVendor.dbvsnowflake || 
               vendor == EDbVendor.dbvsparksql || vendor == EDbVendor.dbvvertica;
    }
    
    private void handleDefaultToken(TSourceToken psourcetoken) {
        psourcetoken.tokentype = ETokenType.ttkeyword;
        
        if (psourcetoken.tokencode < 255) {
            // Single character operators (ASCII characters)
            psourcetoken.setAstext(Character.toString(yylvalstr.charAt(0)));
            psourcetoken.tokentype = ETokenType.ttsinglecharoperator;
            
            switch (psourcetoken.tokencode) {
                case ',':
                    psourcetoken.tokentype = ETokenType.ttcomma;
                    break;
                case '(':
                    psourcetoken.tokentype = ETokenType.ttleftparenthesis;
                    break;
                case ')':
                    psourcetoken.tokentype = ETokenType.ttrightparenthesis;
                    break;
                case '[':
                    psourcetoken.tokentype = ETokenType.ttleftbracket;
                    break;
                case ']':
                    psourcetoken.tokentype = ETokenType.ttrightbracket;
                    break;
                case '.':
                    psourcetoken.tokentype = ETokenType.ttperiod;
                    break;
                case ';':
                    psourcetoken.tokentype = ETokenType.ttsemicolon;
                    break;
                case '$':
                    psourcetoken.tokentype = ETokenType.ttdolorsign;
                    break;
                case ':':
                    psourcetoken.tokentype = ETokenType.ttcolon;
                    break;
                case '+':
                    psourcetoken.tokentype = ETokenType.ttplussign;
                    break;
                case '-':
                    psourcetoken.tokentype = ETokenType.ttminussign;
                    break;
                case '*':
                    psourcetoken.tokentype = ETokenType.ttasterisk;
                    break;
                case '/':
                    psourcetoken.tokentype = ETokenType.ttslash;
                    break;
                case '^':
                    psourcetoken.tokentype = ETokenType.ttcaret;
                    break;
                case '<':
                    psourcetoken.tokentype = ETokenType.ttlessthan;
                    break;
                case '>':
                    psourcetoken.tokentype = ETokenType.ttgreaterthan;
                    break;
                case '=':
                    psourcetoken.tokentype = ETokenType.ttequals;
                    break;
                case '@':
                    if (delimiterchar == '@') {
                        psourcetoken.tokencode = (int)';';
                        psourcetoken.tokentype = ETokenType.ttsemicolon;
                    } else {
                        psourcetoken.tokentype = ETokenType.ttatsign;
                    }
                    break;
                case '~':
                    psourcetoken.tokentype = ETokenType.tttilde;
                    break;
                case '&':
                    psourcetoken.tokentype = ETokenType.ttampersand;
                    break;
                case '|':
                    psourcetoken.tokentype = ETokenType.ttverticalbar;
                    break;
                case '?':
                    if (isqmarktoident && dbvendor != EDbVendor.dbvodbc &&
                        dbvendor != EDbVendor.dbvpostgresql && dbvendor != EDbVendor.dbvgaussdb && dbvendor != EDbVendor.dbvedb && dbvendor != EDbVendor.dbvduckdb) {
                        psourcetoken.tokencode = ident;
                    }
                    break;
            }
        } else if (dbvendor == EDbVendor.dbvhive && psourcetoken.tokencode == TBaseType.hive_equal) {
            psourcetoken.tokentype = ETokenType.ttequals;
        }
    }

}