Source code

001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************}
004{   Lexical analizer for GSQLParser component java version    }
005{   Copyright (c) 2004-2025 by Gudu software              }
006{****************************************************/
007
008package gudusoft.gsqlparser;
009
010import gudusoft.gsqlparser.nodes.TTypeName;
011
012import java.util.HashMap;
013import java.io.InputStreamReader;
014
015import java.util.Locale;
016import java.io.BufferedReader;
017import java.io.IOException;
018
019
020
021public class TLexerMssql extends TCustomLexer{
022
023    static int yynmarks = 0  ;
024    static int yynmatches ;
025    static int yyntrans   ;
026    static int yynstates  ;
027    static int[]  yyk,yym ; // 1 based
028    static int[]  yytint;  // 1 based
029    static TYytRec[] yyt ;  // 1 based
030    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
031    private  static  String[] keywordlist;
032    static String   table_file;
033
034    static HashMap<String, Integer> keywordValueList;
035    static HashMap<Integer, Integer> keywordTypeList;
036        
037        static int[][] yystateTable;
038        
039
040    static {
041        keywordValueList = new HashMap();
042        keywordTypeList = new HashMap();
043        table_file = "/gudusoft/gsqlparser/parser/mssql/mssql_lex_table.txt";
044                if (TBaseType.enterprise_edition||TBaseType.sqlserver_edition||TBaseType.generic_edition){
045                inittable();
046            }
047    }
048
049    public TLexerMssql(){
050          super();
051          dbvendor = EDbVendor.dbvmssql;
052          sourcetokens = new TSourceTokenList();
053    }
054
055
056    public TSourceTokenList sourcetokens;
057
058
059public static boolean canBeColumnName(int tokencode){
060    //http://blog.csdn.net/superbeck/article/details/5387476
061    boolean ret = false;
062    int modifiers = keyword_type_identifier | keyword_type_column ;
063        Integer s = keywordTypeList.get(tokencode);
064    if (s != null){
065                int modifier = s;
066        ret = (modifiers & modifier) == modifier;
067    }
068
069    return ret;
070}
071
072public static boolean checkExpressionContext(String commentString, String lineString) {
073
074  // If the whole line is just this token, treat as a comment
075  if (commentString.equalsIgnoreCase(lineString)) {
076      return false;
077  }
078
079  // e.g. ISNULL(dc.claim_sk,--1) as [claim_sk]
080  // e.g. GROUP BY (CEILING((CAST(((DATEDIFF(DAY, CAST('#' AS DATE), DATEADD(WEEK, DATEDIFF(WEEK, -1, DATEADD(day, --1,[ta_3].[date])), DATEFROMPARTS(-1, -1, -1))) + -1 + -1) - -1) AS float) / NULLIF(CAST(-1.0 AS float),-1.0))) - -1)
081
082  // Find the first non-digit character after "--"
083  String afterHyphens = commentString.substring(2);
084  char suffixChar = 0;
085  int suffixCharPos = -1;
086  for (int i = 0; i < afterHyphens.length(); i++) {
087    char c = afterHyphens.charAt(i);
088    if (!Character.isDigit(c)) {
089      suffixChar = c;
090      suffixCharPos = i;
091      break;
092    }
093  }
094
095  // First non-digit must be ')' or ',' to even be considered an expression
096  if (suffixChar != ')' && suffixChar != ',') {
097      return false;
098  }
099
100  // If first non-digit is ',', check context:
101  // Inside parens (function call): expression (e.g., DATEADD(day, --1,[col]))
102  // Outside parens (select list): comment (e.g., SELECT a, --48232, text)
103  if (suffixChar == ',') {
104      int dashPos = lineString.indexOf("--");
105      if (dashPos > 0) {
106          // No whitespace before "--" is always expression (e.g., ISNULL(x,--1,y))
107          if (!Character.isWhitespace(lineString.charAt(dashPos - 1))) {
108              return true;
109          }
110          // Whitespace before "--": check paren depth to distinguish
111          // function call context from select list context
112          int parenDepth = 0;
113          for (int k = 0; k < dashPos; k++) {
114              char ch = lineString.charAt(k);
115              if (ch == '(') parenDepth++;
116              else if (ch == ')') parenDepth--;
117          }
118          // Inside parens -> expression; outside parens -> comment
119          return parenDepth > 0;
120      }
121      return false;
122  }
123
124  // First non-digit is ')'.
125  // If there's non-whitespace immediately before "--" on the line, it's an expression.
126  // e.g. ISNULL(x,--2) or ISNULL(x,--2) AS col
127  int dashPos = lineString.indexOf("--");
128  if (dashPos > 0 && !Character.isWhitespace(lineString.charAt(dashPos - 1))) {
129      return true;
130  }
131
132  // "--" is at line start or preceded only by whitespace.
133  // Check what follows the ')': if only more ')' and whitespace, it's an expression
134  // in a multi-line context like ISNULL(x,\n  --2).
135  // If there's meaningful text, it's a comment like "--2) some description".
136  String remaining = afterHyphens.substring(suffixCharPos + 1).trim();
137  if (remaining.isEmpty()) {
138      return true;
139  }
140  for (int j = 0; j < remaining.length(); j++) {
141      char rc = remaining.charAt(j);
142      if (rc != ')' && !Character.isWhitespace(rc)) {
143          return false;
144      }
145  }
146  return true;
147
148}
149
150public  int iskeyword(String str){
151       int ret = -1;
152           Integer s = keywordValueList.get(str.toUpperCase(Locale.ENGLISH));
153       if( s != null){
154                 ret = s;
155       }
156       return ret;// -1 means not a keyword
157    }
158
159    public int getkeywordvalue(String keyword){
160        int ret = 0;
161        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
162        if( s != null){
163            ret = s;
164         }
165        return ret;// 0 means not a keyword
166     }
167         
168    public static EKeywordType getKeywordType(String keyword){
169        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
170    }    
171
172    int issystemvariable(String str){
173        return -1;// -1 means not a system variable
174    }
175
176    static void yystateLookupConfigure() {
177        int yystates = yytl.length;
178        yystateTable = new int[257][yystates];
179
180        // initialize to empty
181        for(int i = 0; i < yystates; i++) {
182            for (int j = 0; j < 257; j++)
183                yystateTable[j][i] = -1;
184        }
185
186        for(int i = 0; i < yystates; i++) {
187            int low = yytl[i];
188            int high = yyth[i];
189            for (int j = low; j <= high; j++) {
190                for (char c: yyt[j].cc) {
191                    yystateTable[c][i] = j;
192                }
193            }
194        }
195    }
196        
197        
198    int yylex(){
199          int yyn;
200           while (true) { // top level while
201              yynew();
202              while (true){  //scan
203                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
204                     yymark(yyk[yyn]);
205                 }
206
207                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
208                    yymatch(yym[yyn]);
209                 }
210
211                 if(yytl[yystate] > yyth[yystate]){
212                     break;
213                 }
214
215                 yyscan();
216//                 yyn = yytl[yystate];
217                 totablechar();
218//                 while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
219//                   yyn++;
220//                 }
221//                 if (yyn > yyth[yystate]){
222//                     break;
223//                 }
224
225                 yyn = yystateTable[yytablechar][yystate];
226                 if (yyn == -1)
227                     break;
228
229                                         yystate = yyt[yyn].s;
230              } //scan
231
232              while (true){ //action
233                int yyrule;
234                if ( (yyrule = yyfind()) != -1 ){
235                   yyaction(yyrule);
236                   if (yyreject){
237                       continue;
238                   }
239                }else if( (!yydefault() ) && (yywrap()) ){
240                   yyclear();
241                   returni(0);
242                }
243                break;
244              }
245
246              if (!yydone) {
247                  continue;
248              }
249              break;
250            } // top level while
251
252           return yyretval;
253        }
254
255    static void inittable(){
256                
257                //if (yynmarks > 0) return; //init table already
258
259        String line;
260        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
261        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
262        int c=0;
263        keywordValueList.clear();
264        keywordTypeList.clear();
265        
266        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerMssql.class.getResourceAsStream(table_file)));
267
268            try{
269                while( (line = br.readLine()) != null){
270                          if (line.trim().startsWith("yynmarks=")){
271                             String[] ss = line.split("[=;]");
272                              yynmarks=Integer.parseInt(ss[1].trim());
273                              yyk = new int[yynmarks+1];
274                          }else if (line.trim().startsWith("yynmatches=")){
275                              String[] ss = line.split("[=;]");
276                               yynmatches=Integer.parseInt(ss[1].trim());
277                               yym = new int[yynmatches+1];
278                          }else if (line.trim().startsWith("yyntrans=")){
279                              String[] ss = line.split("[=;]");
280                               yyntrans=Integer.parseInt(ss[1].trim());
281                               yytint = new int[yyntrans+1];
282                               yyt = new TYytRec[yyntrans+1];
283                          }else if (line.trim().startsWith("yynstates=")){
284                              String[] ss = line.split("[=;]");
285                               yynstates=Integer.parseInt(ss[1].trim());
286                               yykl = new int[yynstates];
287                               yykh = new int[yynstates];
288                              yyml = new int[yynstates];
289                              yymh = new int[yynstates];
290                              yytl = new int[yynstates];
291                              yyth = new int[yynstates];
292                          }else if (line.trim().startsWith("<end>")){
293                              if (inyyk){
294                                  inyyk = false;
295                                 if (yynmarks+1 != yyk_count ){
296                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
297                                 }
298                              }
299                              else if(inyym){
300                                     inyym = false;
301                                    if (yynmatches+1 != yym_count ){
302                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
303                                    }
304                              }
305                              else if(inyykl){
306                                     inyykl = false;
307                                    if (yynstates != yykl_count ){
308                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
309                                    }
310                              }
311                              else if(inyykh){
312                                     inyykh = false;
313                                    if (yynstates != yykh_count ){
314                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
315                                    }
316                              }
317                              else if(inyyml){
318                                     inyyml = false;
319                                    if (yynstates != yyml_count ){
320                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
321                                    }
322                              }
323                              else if(inyymh){
324                                     inyymh = false;
325                                    if (yynstates != yymh_count ){
326                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
327                                    }
328                              }
329                              else if(inyytl){
330                                     inyytl = false;
331                                    if (yynstates != yytl_count ){
332                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
333                                    }
334                              }
335                              else if(inyyth){
336                                     inyyth = false;
337                                    if (yynstates != yyth_count ){
338                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
339                                    }
340                              }
341                              else if(inyytint){
342                                     inyytint = false;
343                                    if (yyntrans + 1 != yytint_count ){
344                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
345                                    }
346                              }
347                              else if(inyyt){
348                                     inyyt = false;
349                                    if (yyntrans+1 != yyt_count ){
350                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
351                                    }
352                              }
353                              else if(inkeyword){
354                                     inkeyword = false;
355                              }
356                          }else if(line.trim().startsWith("yyk =")){
357                             inyyk = true; 
358                          }else if(line.trim().startsWith("yym =")){
359                             inyym = true;
360                          }else if(line.trim().startsWith("yykl =")){
361                             inyykl = true;
362                          }else if(line.trim().startsWith("yykh =")){
363                             inyykh = true;
364                          }else if(line.trim().startsWith("yyml =")){
365                             inyyml = true;
366                          }else if(line.trim().startsWith("yymh =")){
367                             inyymh = true;
368                          }else if(line.trim().startsWith("yytl =")){
369                             inyytl = true;
370                          }else if(line.trim().startsWith("yyth =")){
371                             inyyth = true;
372                          }else if(line.trim().startsWith("yytint =")){
373                             inyytint = true;
374                          }else if(line.trim().startsWith("yyt =")){
375                             inyyt = true;
376                        }else if(line.trim().startsWith("keywordsvalue =")){
377                           inkeyword = true;
378                        }else if(inyyk){
379                             String[] ss = line.split("[,]");
380                               for(int j=0;j<ss.length;j++){
381                                   // System.out.println(ss[j].trim());
382                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
383                               }
384                          }else if(inyym){
385                               String[] ss = line.split("[,]");
386                                 for(int j=0;j<ss.length;j++){
387                                     // System.out.println(ss[j].trim());
388                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
389                                 }
390                          }else if(inyykl){
391                               String[] ss = line.split("[,]");
392                                 for(int j=0;j<ss.length;j++){
393                                    //  System.out.println(ss[j].trim());
394                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
395                                 }
396                          }else if(inyykh){
397                               String[] ss = line.split("[,]");
398                                 for(int j=0;j<ss.length;j++){
399                                     // System.out.println(ss[j].trim());
400                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
401                                 }
402                          }else if(inyyml){
403                               String[] ss = line.split("[,]");
404                                 for(int j=0;j<ss.length;j++){
405                                     // System.out.println(ss[j].trim());
406                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
407                                 }
408                          }else if(inyymh){
409                               String[] ss = line.split("[,]");
410                                 for(int j=0;j<ss.length;j++){
411                                     // System.out.println(ss[j].trim());
412                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
413                                 }
414                          }else if(inyytl){
415                               String[] ss = line.split("[,]");
416                                 for(int j=0;j<ss.length;j++){
417                                     // System.out.println(ss[j].trim());
418                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
419                                 }
420                          }else if(inyyth){
421                               String[] ss = line.split("[,]");
422                                 for(int j=0;j<ss.length;j++){
423                                     // System.out.println(ss[j].trim());
424                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
425                                 }
426                          }else if(inyytint){
427                               String[] ss = line.split("[,]");
428                                 for(int j=0;j<ss.length;j++){
429                                     // System.out.println(ss[j].trim());
430                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
431                                 }
432                          }else if(inyyt){
433                                //System.out.println(line.trim());
434
435                              c = 0;
436                              String[] st = line.trim().split(",,");
437                              char[] tmp = new char[st.length];
438                              for(int i=0;i<st.length;i++){
439
440                                  if(st[i].startsWith("\'")) {
441                                      if(st[i].length() == 3){  // 'a'
442                                          tmp[c++] = st[i].charAt(1);
443                                      }else if(st[i].length() == 4) { // '\\'
444                                          tmp[c++] = st[i].charAt(2);
445                                      }else{
446                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
447                                      }
448                                  }else{
449                                      try{
450                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
451                                          } catch (NumberFormatException nfe) {
452                                             System.out.println("NumberFormatException: " + nfe.getMessage());
453                                          }
454                                  }
455                              } //while hasmoreTokens
456
457                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
458                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
459                              yyt_count++;
460
461                          }else if(inkeyword){
462                              String[] ss =line.split("[=]");
463
464                              int val1 = -1;
465                              int val2 = -1;
466                              try {
467                                  val1 = Integer.parseInt(ss[1]);
468                                  val2 = Integer.parseInt(ss[2]);
469                              }
470                              catch (NumberFormatException nfe) {
471                                  System.out.println("NumberFormatException: " + nfe.getMessage());
472                              }
473                              keywordValueList.put(ss[0].toUpperCase(java.util.Locale.ROOT),val1);
474                              keywordTypeList.put(val1,val2);
475                             }
476                }
477                }catch(IOException e){
478                  System.out.println(e.toString());
479                }
480                                
481                                yystateLookupConfigure();
482
483    }
484
485
486
487    void yyaction(int yyruleno){
488
489
490
491      int ic;
492      char[] tmparray = {'=','+','-','*','/','>','<'};
493
494      yylvalstr = getyytext();
495  /* actions: */
496  switch(yyruleno){
497  case 1:
498                
499        {
500           addlit(yylvalstr,yytextlen);
501           if (xcdepth <= 0)
502              {
503               start(init);
504               yylvalstr = litbufdup();
505               returni(cmtslashstar);
506              }
507           else
508              xcdepth--;
509
510         break;
511        }
512
513
514  case 2:
515                
516          {
517              xcdepth++;
518              yyless(2);
519              addlit(yylvalstr,yytextlen);
520              break;
521           }
522
523  case 3:
524                        
525          {
526
527              if (getyysstate() == xq)
528              {
529                  nchars = yytextlen;
530                  addlit(yylvalstr, yytextlen-1);
531                  yyless(nchars-1);
532                  return;//exit;
533              }
534
535              xcdepth = 0;
536              start(xc);
537              startlit();
538              yyless(2);
539              addlit(yylvalstr,yytextlen);
540
541          break;
542          }
543
544  case 4:
545                
546          {
547            addlit(yylvalstr,yytextlen);
548            break;
549          }
550
551  case 5:
552                
553          {
554              addlitchar(yylvalstr.charAt(0));
555              break;
556          }
557
558  case 6:
559                
560          {
561              start(init);
562              addlit(yylvalstr, yytextlen);
563              yylvalstr = litbufdup();
564              returni(sconst);
565                          //System.out.println("stop literal: "+yylvalstr);
566              break;
567          }
568
569  case 7:
570                
571          {
572            addlit(yylvalstr, yytextlen);
573            break;
574          }
575
576  case 8:
577            
578          {
579              if (getyysstate() == xq)
580              {
581                  nchars = yytextlen;
582                  addlit(yylvalstr, yytextlen-1);
583                  yyless(nchars-1);
584                  return;//exit;
585              }
586
587              start(xq);
588              startlit();
589              addlit(yylvalstr,yytextlen);
590              break;
591          }
592
593  case 9:
594                        
595          {
596              if (yysstate == xd)
597                {
598                  addlit(yylvalstr, yytextlen);
599                                  
600                }
601              else
602                {
603                  start(xq);
604                  startlit();
605                  addlit(yylvalstr, yytextlen);
606                                  
607                }
608
609              break;
610          }
611
612  case 10:
613                
614          {
615              addlit(yylvalstr, yytextlen);
616                          //System.out.println("inside literal, found double quote: "+yylvalstr);
617              break;
618          }
619
620  case 11:
621                
622          {
623            dummych1 = get_char();
624            unget_char(dummych1);
625            if (dummych1 == (char)10)
626              {
627                dummych1 = get_char();
628                addlit(yylvalstr+dummych1, yytextlen+1);
629              } 
630                else
631                  addlit(yylvalstr, yytextlen);
632                        //System.out.println("inside literal: "+yylvalstr);
633          break;
634         }
635        
636
637  case 12:
638                
639          {
640              start(init);
641              addlit(yylvalstr, yytextlen);
642              if (literallen == 0)   returni (error);
643              if (literallen >= namedatalen)
644              {
645                 setlengthofliteralbuf(namedatalen);
646                 literallen = namedatalen;
647              }
648              yylvalstr = litbufdup();
649              returni (ident);
650
651              break;
652          }
653
654  case 13:
655               
656          {
657          addlit(yylvalstr, yytextlen);
658          break;
659          }
660
661  case 14:
662                        
663          {
664              start(xd);
665              startlit();
666              addlit(yylvalstr, yytextlen);
667              break;
668          }
669        
670  case 15:
671                
672          {
673            dummych1 = get_char();
674            unget_char(dummych1);
675            if (dummych1 == (char)10)
676              {
677                    dummych1 = get_char();
678                    addlit(yylvalstr+dummych1, yytextlen+1);
679              } else
680                addlit(yylvalstr, yytextlen);
681
682          break;
683          }
684
685  case 16:
686                              
687          {
688              dummych1 = get_char();
689              unget_char(dummych1);
690              if (dummych1 == ']')
691                  {
692                      dummych1 = get_char();
693                      addlit(yylvalstr+dummych1, yytextlen+1);
694                  }
695              else
696                  {
697                      start(init);
698                      addlit(yylvalstr, yytextlen);
699                      if (literallen == 0) returni (error);
700                      if (literallen >= namedatalen)
701                          {
702                              setlengthofliteralbuf(namedatalen);
703                              literallen = namedatalen;
704                          }
705                      yylvalstr = litbufdup();
706                      int rw;
707
708                                                    if ( ((rw = iskeyword(yylvalstr.substring(1, yylvalstr.length() - 1))) != -1)
709                                                                        &&(TTypeName.searchTypeByName(yylvalstr.substring(1, yylvalstr.length() - 1)) != null)
710                                                         )  
711                                                     {
712                                                                        // returni(rw);
713                                                                                if (rw == TBaseType.rrw_date){
714                                                                                        returni(ident);
715                                                                                }else{
716                                                                                        returni(rw);
717                                                                                }                                                                       
718                                                                 }
719                                                    else
720                                                                        returni (ident);
721                  }
722
723          break;
724          }
725
726
727  case 17:
728                               
729          {
730              start(xdbracket);
731              startlit();
732              addlit(yylvalstr, yytextlen);
733              break;
734          }
735        
736  case 18:
737                              
738          {
739              dummych1 = get_char();
740              unget_char(dummych1);
741              if (dummych1 == (char)10)
742              {
743                  dummych1 = get_char();
744                  addlit(yylvalstr+dummych1, yytextlen+1);
745              }
746              else
747                  addlit(yylvalstr, yytextlen);
748
749          break;
750          }
751
752  case 19:
753                          
754          {
755              dummych1 = get_char();
756              unget_char(dummych1);
757              if (dummych1 == '}')
758                  {
759                      dummych1 = get_char();
760                      addlit(yylvalstr+dummych1, yytextlen+1);
761                  }
762              else
763                  {
764                      start(init);
765                      addlit(yylvalstr, yytextlen);
766                      if (literallen == 0)  returni (error);
767                      yylvalstr = litbufdup();
768                      returni (ident);
769                  }
770
771          break;
772          }
773
774
775  case 20:
776                             
777          {
778              start(xdbrace);
779              startlit();
780              addlit(yylvalstr, yytextlen);
781              break;
782          }
783        
784  case 21:
785                          
786          {
787              dummych1 = get_char();
788              unget_char(dummych1);
789              if (dummych1 == (char)10)
790              {
791                  dummych1 = get_char();
792                  addlit(yylvalstr+dummych1, yytextlen+1);
793              }
794              else
795                  addlit(yylvalstr, yytextlen);
796
797          break;
798          }
799
800  case 22:
801                
802          {
803            returni(lexnewline);
804            break;
805          }
806
807  case 23:
808        
809          {
810              returni(lexspace);
811              break;
812          }
813
814  case 24:
815                
816          {
817          if ((getyysstate() == xq)
818              || (getyysstate() == xd)
819              || (getyysstate() == xc)
820              || (getyysstate() == xdbracket))
821          {
822              addlit(yylvalstr, 1);
823              yyless(1);
824              return;//exit;
825          }
826
827          // Check if it's actually a comment or a numeric operation
828          // 1. If the third char is a space, it's definitely a comment
829          // 2. If the third char is a digit, then we need to check whether this token is at the beginning of a line,
830          //    if it is, then it's a comment, otherwise it's a numeric operation
831          if (yylvalstr.length() > 2) {
832              char thirdChar = yylvalstr.charAt(2);
833              if (Character.isWhitespace(thirdChar)) {
834                  // If the third char is a space, it's definitely a comment
835                  returni(cmtdoublehyphen);
836                  break;
837              } else if (Character.isDigit(thirdChar)) {
838                  // If the third char is a digit, check the previous token
839                  // Check if we're in an expression context (after identifier or number)
840                  // abc--123, 234--abc, 123--123, 123--abc, abc--abc, all are valid arithmetic operations
841                  boolean inExpressionContext = false;
842
843                  // Need to check if the previous token was an identifier or number.
844                  // Only the current line counts: the line buffer is reused, and text
845                  // left after it by a longer earlier line must not change the verdict.
846                  if (checkExpressionContext(yylvalstr.trim(),currentLineText().trim())) {
847                    inExpressionContext = true;
848                  }
849
850                  if (isAtBeginOfLine() || !inExpressionContext) {
851                    // Treat as a comment if at start of line or not in expression
852                    returni(cmtdoublehyphen);
853                    break;
854                  } else {
855                    // We're in an expression context, treat as mathematical operator
856                    yyless(1);  // Only take the first '-'
857                    returnc('-');
858                  }                  
859                  break;
860              }
861              // For any other character, it's still a comment
862          }
863
864          returni(cmtdoublehyphen);
865          break;
866          }
867
868  case 25:
869            
870        {
871      returni(twocolons);
872      break;
873    }
874
875
876  case 26:
877      
878          {
879               returnc(yylvalstr.charAt(0));
880               break;
881          }
882
883  case 27:
884                       
885          {
886            returni(cmpop);
887            break;
888          }
889        
890  case 28:
891           
892          {
893
894          if (getyysstate() == xc)
895             {
896              slashstar = yylvalstr.indexOf("*/");
897                if (slashstar >= 0)
898                  {
899                      addlit(yylvalstr,slashstar+2);
900                      yylvalstr = litbufdup();
901                      yyless(slashstar+2);
902
903                        if (xcdepth <= 0){
904                      start(init);
905                      returni(cmtslashstar);
906                        }else{
907                                xcdepth--;
908                        }
909                  }
910                else
911                  {
912                      addlit(yylvalstr,1);
913                      yyless(1);
914                  }
915             }
916          else
917            {
918              nchars = yytextlen;
919              slashstar = yylvalstr.indexOf("/*");
920              dashdash = yylvalstr.indexOf("--");
921              if ((slashstar >= 0) && (dashdash >= 0))
922                {
923                  //if both appear, take the first one
924                   if (slashstar > dashdash)
925                    {slashstar = dashdash;}
926                }
927              else
928                {
929                  //   if slashstar=0 then slashstar := dashdash;
930                  // add (getyysstate <> xc) to avoid something like this */--,here */ should be handled instead of --
931                  if ((slashstar > 0) && (getyysstate() != xc)) {
932                    nchars = slashstar;
933                    }
934                }
935
936              while ((nchars > 1)
937                      && ( (yylvalstr.charAt(nchars-1) == '+' )
938                          || (yylvalstr.charAt(nchars-1) =='-'))
939                      && (getyysstate() != xc))
940                {
941                  for (ic = nchars - 1; ic>=1; ic--)
942                  {
943                    if (isopchar(yylvalstr.charAt(ic-1)))  break;
944                  }
945                  if (ic >= 1) break;
946                  nchars--;
947                }
948
949              if (nchars < yytextlen)
950                {
951                  //Strip the unwanted chars from the token
952                  yyless(nchars);
953                  yylvalstr = yylvalstr.substring(0,nchars);
954                }
955
956                  ///*
957                  // * If what we have left is only one char, and it's
958                  // * one of the characters matching "self", then
959                  // * return it as a character token the same way
960                  // * that the "self" rule would have.
961                  // * make sure @ return as self char, by james wang
962                  // */
963                  if ((nchars == 1) && (isselfchar(yylvalstr.charAt(0)) || (yylvalstr.charAt(0) == '@')))
964                    {
965                      returnc(yylvalstr.charAt(0));
966                    }
967                  else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1-1) == '.') )
968                    {
969                        yyless(1);
970                        returnc(yylvalstr.charAt(0));
971                    }
972                  else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '?') )
973                    {
974                        yyless(1);
975                        returnc(yylvalstr.charAt(0));
976                    }
977                  else if ( ((yylvalstr.charAt(0) == '+')||(yylvalstr.charAt(0) == '-')||(yylvalstr.charAt(0) == '*')||(yylvalstr.charAt(0) == '/')||(yylvalstr.charAt(0) == '%') ))
978                    {
979                                          if (nchars >= 2){
980                                                if (yylvalstr.charAt(1) == '='){
981                                                        yyless(2);
982                                                        yylvalstr = yylvalstr.substring(0,2);
983                                                        returni(op);
984                                                }else{
985                                                        yyless(1);
986                                                        returnc(yylvalstr.charAt(0));
987                                                }
988                                          }else{
989                                                returnc(yylvalstr.charAt(0));
990                                          }
991                        
992                    }
993                  else if (
994                              (nchars >= 2)
995                                  &&(
996                                       charinarray(yylvalstr.charAt(nchars-1-1), tmparray)
997                                       && (yylvalstr.charAt(nchars-1) == ':')
998                                       )
999                           )
1000                      {
1001                        yyless(nchars-1);
1002                         yylvalstr = yylvalstr.substring(0,nchars-1);
1003                        if (nchars == 2)
1004                          returnc(yylvalstr.charAt(0));
1005                        else
1006                          returni(op);
1007                      }
1008                    else if (
1009                                (nchars >= 2)
1010                                    && (
1011                                          charinarray(yylvalstr.charAt(nchars-1-1),tmparray)
1012                                          && (yylvalstr.charAt(nchars-1) == '.')
1013                                         )
1014                               )
1015                        {
1016                          yyless(nchars-1);
1017                           yylvalstr = yylvalstr.substring(0,nchars-1);
1018                          if (nchars == 2)
1019                            returnc(yylvalstr.charAt(0));
1020                          else
1021                            returni(op);
1022                        }
1023                    else if (
1024                                (nchars > 2)
1025                                    && ((yylvalstr.charAt(nchars-1) == '-')
1026                                        ||(yylvalstr.charAt(nchars-1) == '+')
1027                                         )
1028                               )
1029                        {
1030                          yyless(nchars-1);
1031                           yylvalstr = yylvalstr.substring(0,nchars-1);
1032                           returni(op);
1033                        }
1034                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == '@') )
1035                      {
1036                        if (nchars == 2)
1037                          { // such as >@
1038                              yyless(nchars-1);
1039                            yylvalstr = yylvalstr.substring(0,nchars-1);
1040                             returnc(yylvalstr.charAt(0));
1041                          }
1042                        else
1043                          {
1044                            if ((yylvalstr.charAt(nchars-1-1) == '@')||(yylvalstr.charAt(nchars-1-1) == '-'))
1045                              { //such as >@@,>=@@, =-@
1046                                  yyless(nchars-2);
1047                                yylvalstr = yylvalstr.substring(0,nchars-2);
1048                                if (nchars == 3)
1049                                    returnc(yylvalstr.charAt(0));
1050                                else
1051                                    returni(cmpop);
1052                              }
1053                            else
1054                              { // >=@
1055                                                                yyless(nchars-1);
1056                                                                yylvalstr = yylvalstr.substring(0,nchars-1);
1057                                                                returni(cmpop);
1058                              }
1059                          }
1060                         }
1061                      else if ( (nchars >= 2) && (yylvalstr.charAt(0)== '|') && (yylvalstr.charAt(1) == '|'))
1062                        { //||--sss ,just get || only
1063                              yyless(2);
1064                              yylvalstr = yylvalstr.substring(0,2);
1065                              returni(concatenationop); //return as concatenationop, this is used in sybase only
1066                        }
1067                      else if ((nchars >= 2) && ((yylvalstr.charAt(0) =='.') && ( (yylvalstr.charAt(1)=='*') || (yylvalstr.charAt(1) =='#')) ))
1068                          {
1069                            yyless(1);
1070                            returnc(yylvalstr.charAt(0));
1071                          }
1072                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '#')) ))
1073                          {
1074                            yyless(1);
1075                            returnc(yylvalstr.charAt(0));
1076                          }
1077                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '?')) ))
1078                          {
1079                            yyless(1);
1080                            returnc(yylvalstr.charAt(0));
1081                          }
1082                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '#') && ( (yylvalstr.charAt(1) == '.')) ))
1083                          {
1084                            yyless(1);
1085                            returnc(yylvalstr.charAt(0));
1086                          }
1087                      else if ((nchars == 2) && (((yylvalstr.charAt(0) == '>')||(yylvalstr.charAt(0) == '<')) && ( (yylvalstr.charAt(1) == '#')) ))
1088                          {
1089                            yyless(1);
1090                            returnc(yylvalstr.charAt(0));
1091                          }
1092                      else if ((nchars == 3) && ((yylvalstr.charAt(1) == '#') && ( (yylvalstr.charAt(2) == '.')) ))
1093                          {
1094                            yyless(1);
1095                            returnc(yylvalstr.charAt(0));
1096                          }
1097                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '$') && ( (yylvalstr.charAt(1) == '-')) ))
1098                          {
1099                            yyless(1);
1100                            returnc(yylvalstr.charAt(0));
1101                          }
1102                                        else if ( (nchars >= 3) && (yylvalstr.charAt(nchars-1) == '$' ) && (yylvalstr.charAt(nchars-2) == '$' ))
1103                                        { // =$$abc
1104                                            if (nchars == 3){
1105                                                        yyless(1);
1106                                                        returnc(yylvalstr.charAt(0));
1107                                                }else{
1108                                                        yyless(nchars-2);
1109                                                        yylvalstr = yylvalstr.substring(0,nchars-2);
1110                                                        returni(cmpop);
1111                                                }
1112                                        }
1113                                          else if (((nchars > 2) && (yylvalstr.charAt(0) == '*'))
1114                              && (yylvalstr.charAt(1) == '/')
1115                              && (getyysstate() == xc)
1116                              )
1117                           { //in comment, and find */ , then it must the end of comment
1118                             yyless(2);
1119                                addlit(yylvalstr,yytextlen);
1120                             if (xcdepth <= 0)
1121                               {
1122                                 start(init);
1123                                 yylvalstr = litbufdup();
1124                                 returni(cmtslashstar);
1125                               }
1126                             else
1127                               xcdepth--;
1128                           }
1129                      else
1130                        returni(op);
1131               }
1132
1133          break;
1134          }
1135
1136  case 29:
1137                        
1138          {
1139               returni(iconst);
1140               break;
1141          }
1142
1143  case 30:
1144                        
1145          {
1146            ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
1147            nchars = yytextlen;
1148            if (yylvalstr.charAt(nchars-1) == '.')
1149              {
1150                dummych1 = get_char();
1151                unget_char(dummych1);
1152                if (dummych1 == '.')
1153                  {
1154                      yyless(nchars-1);
1155                    yylvalstr = yylvalstr.substring(0,nchars - 1);
1156                    returni (iconst);
1157                    return;//exit;
1158                  }
1159              }
1160            returni (fconst);
1161           break;
1162          }
1163
1164  case 31:
1165                        
1166          {
1167           returni (fconst);
1168           break;
1169          }
1170
1171  case 32:
1172              
1173          {
1174            int rw;
1175            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1176            else
1177               returni(ident);
1178            break;
1179          }
1180
1181  case 33:
1182           
1183          {
1184            int rw;
1185            if ( (rw = issystemvariable(yylvalstr)) != -1)  { returni(rw);}
1186            else if (yylvalstr.charAt(0) == '$'){
1187                returni(ident);
1188            }
1189            else
1190               returni(variable);
1191
1192            break;
1193          }
1194 
1195
1196
1197  case 34:
1198           
1199          {
1200            int rw;
1201            dummych1 = get_char();
1202            unget_char(dummych1);
1203
1204                        if (dummych1 == ':')
1205                        {
1206                                nchars = yytextlen;
1207                                yyless(nchars-1);
1208                                yylvalstr = yylvalstr.substring(0,nchars-1);
1209                                
1210                    if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1211                    else
1212                       returni(ident);
1213                        }
1214                        else
1215                        {
1216                    returni(mslabel);
1217                        }
1218
1219            break;
1220          }
1221
1222      
1223  case 35:
1224         
1225          {
1226           returni(ident);
1227           break;
1228          }
1229    
1230  case 36:
1231          
1232          {
1233           returni(ident);
1234           break;
1235          }
1236                  
1237  case 37:
1238                        
1239          {
1240            returni( error);
1241                        //System.out.println("found error char");
1242            break;
1243          }
1244    default:{
1245     System.out.println("fatal error in yyaction");
1246    }
1247   }//switch
1248}/*yyaction*/;
1249
1250
1251
1252        }