Source code

001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************}
004{   Lexical analizer for GSQLParser component java version    }
005{   Copyright (c) 2004-2018 by Gudu software              }
006{****************************************************/
007
008package gudusoft.gsqlparser;
009
010import java.util.HashMap;
011import java.io.InputStreamReader;
012
013import java.util.Locale;
014import java.io.BufferedReader;
015import java.io.IOException;
016
017public class TLexerSparksql extends TCustomLexer{
018
019    static int yynmarks = 0  ;
020    static int yynmatches ;
021    static int yyntrans   ;
022    static int yynstates  ;
023    static int[]  yyk,yym ; // 1 based
024    static int[]  yytint;  // 1 based
025    static TYytRec[] yyt ;  // 1 based
026    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
027    private  static  String[] keywordlist;
028    static String   table_file;
029    static HashMap<String, Integer> keywordValueList;
030    static HashMap<Integer, Integer> keywordTypeList;    
031    HashMap sparksqlCharsets;
032        static int[][] yystateTable;
033
034    static {
035              keywordValueList = new HashMap<String, Integer>();
036              keywordTypeList = new HashMap<Integer, Integer>();              
037          table_file = "/gudusoft/gsqlparser/parser/sparksql/sparksql_lex_table.txt";
038                  if (TBaseType.enterprise_edition||TBaseType.sparksql_edition){
039                inittable();
040          }
041    }
042
043
044    public TLexerSparksql(){
045          super();
046          dbvendor = EDbVendor.dbvsparksql;
047          sourcetokens = new TSourceTokenList();
048          sparksqlCharsets  = new HashMap();
049    }
050
051
052    public TSourceTokenList sourcetokens;
053
054    boolean issparksqlCharset(String astr){
055        boolean ret = false;
056        if (sparksqlCharsets.size() == 0){
057            sparksqlCharsets.put("big5","1");
058            sparksqlCharsets.put("dec8","1");
059            sparksqlCharsets.put("cp850","1");
060            sparksqlCharsets.put("hp8","1");
061            sparksqlCharsets.put("koi8r","1");
062            sparksqlCharsets.put("latin1","1");
063            sparksqlCharsets.put("latin2","1");
064            sparksqlCharsets.put("swe7","1");
065            sparksqlCharsets.put("ascii","1");
066            sparksqlCharsets.put("ujis","1");
067            sparksqlCharsets.put("sjis","1");
068            sparksqlCharsets.put("hebrew","1");
069            sparksqlCharsets.put("tis620","1");
070            sparksqlCharsets.put("euckr","1");
071            sparksqlCharsets.put("koi8u","1");
072            sparksqlCharsets.put("gb2312","1");
073            sparksqlCharsets.put("greek","1");
074            sparksqlCharsets.put("cp1250","1");
075            sparksqlCharsets.put("gbk","1");
076            sparksqlCharsets.put("latin5","1");
077            sparksqlCharsets.put("armscii","1");
078            sparksqlCharsets.put("utf8","1");
079            sparksqlCharsets.put("ucs2","1");
080            sparksqlCharsets.put("cp866","1");
081            sparksqlCharsets.put("keybcs2","1");
082            sparksqlCharsets.put("macce","1");
083            sparksqlCharsets.put("macroma","1");
084            sparksqlCharsets.put("cp852","1");
085            sparksqlCharsets.put("latin7","1");
086            sparksqlCharsets.put("utf8mb4","1");
087            sparksqlCharsets.put("cp1251","1");
088            sparksqlCharsets.put("utf16","1");
089            sparksqlCharsets.put("cp1256","1");
090            sparksqlCharsets.put("cp1257","1");
091            sparksqlCharsets.put("utf32","1");
092            sparksqlCharsets.put("binary","1");
093            sparksqlCharsets.put("geostd8","1");
094            sparksqlCharsets.put("cp932","1");
095            sparksqlCharsets.put("eucjpms","1");
096        }
097
098        try{
099            ret =   (Integer.parseInt( (String) sparksqlCharsets.get(astr.toLowerCase()) ) ) == 1 ;
100        }catch(NumberFormatException e){
101            ret = false;
102        }
103
104        return ret;
105    }
106    
107public boolean canBeColumnName(int tokencode){
108    //http://blog.csdn.net/superbeck/article/details/5387476
109    boolean ret = false;
110    int modifiers = keyword_type_identifier | keyword_type_column ;
111    
112    Integer s = keywordTypeList.get(tokencode);
113    if (s != null){
114        int modifier = s;
115        ret = (modifiers & modifier) == modifier;
116    }
117
118    return ret;
119}
120
121public  int iskeyword(String str){
122       int ret = -1;
123       
124       Integer s = keywordValueList.get(str.toUpperCase(Locale.ENGLISH));
125       if( s != null){
126                                        ret = s;                 
127       }
128       return ret;// -1 means not a keyword
129    }
130
131    public int getkeywordvalue(String keyword){
132        int ret = 0;
133        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
134        if( s != null){
135            ret = s;
136        }
137        return ret;// 0 means not a keyword
138    }
139        
140    public static EKeywordType getKeywordType(String keyword){
141        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
142    }           
143
144
145    int issystemvariable(String str){
146        return -1;// -1 means not a system variable
147    }
148        
149    static void yystateLookupConfigure() {
150        int yystates = yytl.length;
151        yystateTable = new int[257][yystates];
152
153        // initialize to empty
154        for(int i = 0; i < yystates; i++) {
155            for (int j = 0; j < 257; j++)
156                yystateTable[j][i] = -1;
157        }
158
159        for(int i = 0; i < yystates; i++) {
160            int low = yytl[i];
161            int high = yyth[i];
162            for (int j = low; j <= high; j++) {
163                for (char c: yyt[j].cc) {
164                    yystateTable[c][i] = j;
165                }
166            }
167        }
168    }   
169
170    int yylex(){
171          int yyn;
172           while (true) { // top level while
173              yynew();
174              while (true){  //scan
175                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
176                     yymark(yyk[yyn]);
177                 }
178
179                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
180                    yymatch(yym[yyn]);
181                 }
182
183                 if(yytl[yystate] > yyth[yystate]){
184                     break;
185                 }
186
187                 yyscan();
188                 //yyn = yytl[yystate];
189                 totablechar();
190                                 
191                 //while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
192                 //  yyn++;
193                 // }
194                 //if (yyn > yyth[yystate]){
195                 //    break;
196                 //}
197
198                 yyn = yystateTable[yytablechar][yystate];
199                 if (yyn == -1)
200                     break;
201                                         
202                 yystate = yyt[yyn].s;
203              } //scan
204
205              while (true){ //action
206                int yyrule;
207                if ( (yyrule = yyfind()) != -1 ){
208                   yyaction(yyrule);
209                   if (yyreject){
210                       continue;
211                   }
212                }else if( (!yydefault() ) && (yywrap()) ){
213                   yyclear();
214                   returni(0);
215                }
216                break;
217              }
218
219              if (!yydone) {
220                  continue;
221              }
222              break;
223            } // top level while
224
225           return yyretval;
226        }
227
228    static void inittable(){
229                
230                //if (yynmarks > 0) return; //init table already
231
232        String line;
233        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
234        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
235        int c=0;
236        keywordValueList.clear();
237        keywordTypeList.clear();
238        
239        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerSparksql.class.getResourceAsStream(table_file)));
240
241            try{
242                while( (line = br.readLine()) != null){
243                          if (line.trim().startsWith("yynmarks=")){
244                             String[] ss = line.split("[=;]");
245                              yynmarks=Integer.parseInt(ss[1].trim());
246                              yyk = new int[yynmarks+1];
247                          }else if (line.trim().startsWith("yynmatches=")){
248                              String[] ss = line.split("[=;]");
249                               yynmatches=Integer.parseInt(ss[1].trim());
250                               yym = new int[yynmatches+1];
251                          }else if (line.trim().startsWith("yyntrans=")){
252                              String[] ss = line.split("[=;]");
253                               yyntrans=Integer.parseInt(ss[1].trim());
254                               yytint = new int[yyntrans+1];
255                               yyt = new TYytRec[yyntrans+1];
256                          }else if (line.trim().startsWith("yynstates=")){
257                              String[] ss = line.split("[=;]");
258                               yynstates=Integer.parseInt(ss[1].trim());
259                               yykl = new int[yynstates];
260                               yykh = new int[yynstates];
261                              yyml = new int[yynstates];
262                              yymh = new int[yynstates];
263                              yytl = new int[yynstates];
264                              yyth = new int[yynstates];
265                          }else if (line.trim().startsWith("<end>")){
266                              if (inyyk){
267                                  inyyk = false;
268                                 if (yynmarks+1 != yyk_count ){
269                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
270                                 }
271                              }
272                              else if(inyym){
273                                     inyym = false;
274                                    if (yynmatches+1 != yym_count ){
275                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
276                                    }
277                              }
278                              else if(inyykl){
279                                     inyykl = false;
280                                    if (yynstates != yykl_count ){
281                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
282                                    }
283                              }
284                              else if(inyykh){
285                                     inyykh = false;
286                                    if (yynstates != yykh_count ){
287                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
288                                    }
289                              }
290                              else if(inyyml){
291                                     inyyml = false;
292                                    if (yynstates != yyml_count ){
293                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
294                                    }
295                              }
296                              else if(inyymh){
297                                     inyymh = false;
298                                    if (yynstates != yymh_count ){
299                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
300                                    }
301                              }
302                              else if(inyytl){
303                                     inyytl = false;
304                                    if (yynstates != yytl_count ){
305                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
306                                    }
307                              }
308                              else if(inyyth){
309                                     inyyth = false;
310                                    if (yynstates != yyth_count ){
311                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
312                                    }
313                              }
314                              else if(inyytint){
315                                     inyytint = false;
316                                    if (yyntrans + 1 != yytint_count ){
317                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
318                                    }
319                              }
320                              else if(inyyt){
321                                     inyyt = false;
322                                    if (yyntrans+1 != yyt_count ){
323                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
324                                    }
325                              }
326                              else if(inkeyword){
327                                     inkeyword = false;
328                              }
329                          }else if(line.trim().startsWith("yyk =")){
330                             inyyk = true; 
331                          }else if(line.trim().startsWith("yym =")){
332                             inyym = true;
333                          }else if(line.trim().startsWith("yykl =")){
334                             inyykl = true;
335                          }else if(line.trim().startsWith("yykh =")){
336                             inyykh = true;
337                          }else if(line.trim().startsWith("yyml =")){
338                             inyyml = true;
339                          }else if(line.trim().startsWith("yymh =")){
340                             inyymh = true;
341                          }else if(line.trim().startsWith("yytl =")){
342                             inyytl = true;
343                          }else if(line.trim().startsWith("yyth =")){
344                             inyyth = true;
345                          }else if(line.trim().startsWith("yytint =")){
346                             inyytint = true;
347                          }else if(line.trim().startsWith("yyt =")){
348                             inyyt = true;
349                        }else if(line.trim().startsWith("keywordsvalue =")){
350                           inkeyword = true;
351                        }else if(inyyk){
352                             String[] ss = line.split("[,]");
353                               for(int j=0;j<ss.length;j++){
354                                   // System.out.println(ss[j].trim());
355                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
356                               }
357                          }else if(inyym){
358                               String[] ss = line.split("[,]");
359                                 for(int j=0;j<ss.length;j++){
360                                     // System.out.println(ss[j].trim());
361                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
362                                 }
363                          }else if(inyykl){
364                               String[] ss = line.split("[,]");
365                                 for(int j=0;j<ss.length;j++){
366                                    //  System.out.println(ss[j].trim());
367                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
368                                 }
369                          }else if(inyykh){
370                               String[] ss = line.split("[,]");
371                                 for(int j=0;j<ss.length;j++){
372                                     // System.out.println(ss[j].trim());
373                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
374                                 }
375                          }else if(inyyml){
376                               String[] ss = line.split("[,]");
377                                 for(int j=0;j<ss.length;j++){
378                                     // System.out.println(ss[j].trim());
379                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
380                                 }
381                          }else if(inyymh){
382                               String[] ss = line.split("[,]");
383                                 for(int j=0;j<ss.length;j++){
384                                     // System.out.println(ss[j].trim());
385                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
386                                 }
387                          }else if(inyytl){
388                               String[] ss = line.split("[,]");
389                                 for(int j=0;j<ss.length;j++){
390                                     // System.out.println(ss[j].trim());
391                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
392                                 }
393                          }else if(inyyth){
394                               String[] ss = line.split("[,]");
395                                 for(int j=0;j<ss.length;j++){
396                                     // System.out.println(ss[j].trim());
397                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
398                                 }
399                          }else if(inyytint){
400                               String[] ss = line.split("[,]");
401                                 for(int j=0;j<ss.length;j++){
402                                     // System.out.println(ss[j].trim());
403                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
404                                 }
405                          }else if(inyyt){
406                                //System.out.println(line.trim());
407
408                              c = 0;
409                              String[] st = line.trim().split(",,");
410                              char[] tmp = new char[st.length];
411                              for(int i=0;i<st.length;i++){
412
413                                  if(st[i].startsWith("\'")) {
414                                      if(st[i].length() == 3){  // 'a'
415                                          tmp[c++] = st[i].charAt(1);
416                                      }else if(st[i].length() == 4) { // '\\'
417                                          tmp[c++] = st[i].charAt(2);
418                                      }else{
419                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
420                                      }
421                                  }else{
422                                      try{
423                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
424                                          } catch (NumberFormatException nfe) {
425                                             System.out.println("NumberFormatException: " + nfe.getMessage());
426                                          }
427                                  }
428                              } //while hasmoreTokens
429
430                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
431                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
432                              yyt_count++;
433
434                          }else if(inkeyword){
435                              String[] ss =line.split("[=]");
436
437                              int val1 = -1;
438                              int val2 = -1;
439                              try {
440                                  val1 = Integer.parseInt(ss[1]);
441                                  val2 = Integer.parseInt(ss[2]);
442                              }
443                              catch (NumberFormatException nfe) {
444                                  System.out.println("NumberFormatException: " + nfe.getMessage());
445                              }
446                              keywordValueList.put(ss[0].toUpperCase(),val1);
447                              keywordTypeList.put(val1,val2);
448
449                             }
450                }
451                }catch(IOException e){
452                  System.out.println(e.toString());
453                }
454                                
455                                yystateLookupConfigure();
456
457    }
458
459
460
461    void yyaction(int yyruleno){
462
463
464      int ic;
465      char[] tmparray = {'=','+','-','*','/','>','<'};
466
467      yylvalstr = getyytext();
468  /* actions: */
469  switch(yyruleno){
470  case 1:
471                
472        {
473                    addlit(yylvalstr,yytextlen);
474                    if (xcdepth <= 0)
475                       {
476                        start(init);
477                        yylvalstr = litbufdup();
478                        returni(cmtslashstar);
479                       }
480                    else
481                       xcdepth--;
482
483                  break;
484        }
485
486
487  case 2:
488                
489                  {
490                         xcdepth++;
491                          yyless(2);
492                          addlit(yylvalstr,yytextlen);
493                      break;
494                   }
495
496  case 3:
497                        
498                  {
499
500                      if (getyysstate() == xq)
501                      {
502                          nchars = yytextlen;
503                          addlit(yylvalstr, yytextlen-1);
504                          yyless(nchars-1);
505                          return;//exit;
506                      }
507
508                      xcdepth = 0;
509                      start(xc);
510                      startlit();
511                      yyless(2);
512                      addlit(yylvalstr,yytextlen);
513
514                  break;
515                  }
516
517  case 4:
518                
519                  {
520                    addlit(yylvalstr,yytextlen);
521                    break;
522                  }
523
524  case 5:
525                
526                  {
527                      addlitchar(yylvalstr.charAt(0));
528                      break;
529                  }
530
531
532  case 6:
533            
534        {
535                                        //token_start := yylvalStr;
536                                        if (getyysstate() == xq)
537            {
538                                                
539                                              nchars = yytextlen;
540                                              addlit(yylvalstr, yytextlen-1);
541                yyless(nchars-1);
542                                              return;//exit;    
543                                           }
544            start(xq);
545            startlit();
546                                          addlit(yylvalstr,yytextlen);
547                break;
548        }
549
550
551  case 7:
552                       
553        {
554                                        start(init);
555                                        addlit(yylvalstr, yytextlen);
556                                        yylvalstr = litbufdup();
557                                        returni(ident);
558                break;
559        }
560
561  case 8:
562                        
563        {
564          if ((getyysstate() == xd)||(getyysstate() == xq))
565          {
566              addlit(yylvalstr, 1);
567              yyless(1);
568              return;//exit;
569          }     
570                                        //token_start := yylvalStr;
571                                        start(xbacktick);
572                                        startlit();
573                                        addlit(yylvalstr, yytextlen);
574                break;
575        }
576
577
578  case 9:
579                       
580        {
581                addlit(yylvalstr, yytextlen);
582                break;
583  }
584
585  case 10:
586                       
587        {
588    addlit(yylvalstr, yytextlen);
589                break;
590   }
591
592
593  case 11:
594                
595                  {
596                      start(init);
597                      addlit(yylvalstr, yytextlen);
598                      yylvalstr = litbufdup();
599                      returni(sconst);
600                      break;
601                  }
602
603  case 12:
604                
605                    {
606                      addlit(yylvalstr,yytextlen);
607                      break;
608                    }
609        
610  case 13:
611                        
612                {
613
614          if (
615              (getyysstate() == xd)
616                                                )
617          {
618              addlit(yylvalstr, 1);
619              yyless(1);
620              return;//exit;
621          }
622          
623                        start(xq);
624                        startlit();
625                        addlit(yylvalstr, yytextlen);
626                        
627                        break;
628                }
629
630
631  case 14:
632                
633                  {
634                      addlit(yylvalstr, yytextlen);
635                      break;
636                  }
637
638  case 15:
639                
640                  {
641                    dummych1 = get_char();
642                    unget_char(dummych1);
643                      while (dummych1 == '\\'){
644                          dummych1 = get_char();
645                          dummych2 = get_char();
646                          addlit(yylvalstr+dummych1+dummych2, yytextlen+2);
647                          yylvalstr = "";
648                          yytextlen = 0;
649    
650                          dummych1 = get_char();
651                          unget_char(dummych1);
652                      }
653                                          
654                    if (dummych1 == (char)10)
655                      {
656                        if (insqlpluscmd){
657                          nchars = yytextlen;
658                          if(yylvalstr.charAt(nchars-1) == (char)13){
659                              yyless(nchars - 1);
660                              yylvalstr = yylvalstr.substring(0,nchars);
661                          }
662                            start(init);
663                            addlit(yylvalstr, nchars-1);
664                            yylvalstr = litbufdup();
665                            returni(sconst); //in sqlplus command, characters between ' and return is treated as a string
666
667                        }else{
668                        dummych1 = get_char();
669                        addlit(yylvalstr+dummych1, yytextlen+1);
670                        }
671                      } 
672                        else
673                            { addlit(yylvalstr, yytextlen);}
674
675                  break;
676                 }
677                                 
678  case 16:
679                 
680        {
681                addlit(yylvalstr, yytextlen);
682                break;
683        }
684
685
686  case 17:
687                
688                  {
689                      start(init);
690                      addlit(yylvalstr, yytextlen);
691                      if ((literallen == 0) && (!insqlpluscmd))
692                        {returni (error);}
693                     // if (literallen >= namedatalen)
694                     // {
695                     //    setlengthofliteralbuf(namedatalen);
696                     //    literallen = namedatalen;
697                     // }
698                      yylvalstr = litbufdup();
699                      returni (sconst);
700
701                      break;
702                  }
703
704  case 18:
705               
706                  {
707                      addlit(yylvalstr, yytextlen);
708                      break;
709                  }
710
711  case 19:
712                        
713                  {
714                      start(xd);
715                      startlit();
716                      addlit(yylvalstr, yytextlen);
717                      break;
718                  }
719  case 20:
720                
721                  {
722                    dummych1 = get_char();
723                    unget_char(dummych1);
724                    if (dummych1 == (char)10)
725                      {
726                          if (insqlpluscmd){
727                            nchars = yytextlen;
728                            if(yylvalstr.charAt(nchars-1) == (char)13){
729                                yyless(nchars - 1);
730                                yylvalstr = yylvalstr.substring(0,nchars);
731                            }
732                              start(init);
733                              addlit(yylvalstr, nchars-1);
734                              yylvalstr = litbufdup();
735                              returni(sconst); //in sqlplus command, characters between ' and return is treated as a string
736
737                          }else{
738                          dummych1 = get_char();
739                          addlit(yylvalstr+dummych1, yytextlen+1);
740                          }
741
742                      } else
743                        addlit(yylvalstr, yytextlen);
744
745                  break;
746                  }
747
748
749
750  case 21:
751          
752          {
753            returni(lexnewline);
754            break;
755          }
756
757  case 22:
758        
759          {
760              returni(lexspace);
761              break;
762          }
763
764  case 23:
765                
766          {
767          if ((getyysstate() == xq)
768              || (getyysstate() == xd)
769              || (getyysstate() == xc)
770              || (getyysstate() == xdbracket)
771              || (getyysstate() == xbacktick)
772              )
773          {
774              addlit(yylvalstr, 1);
775              yyless(1);
776              return;//exit;
777          }
778           
779
780
781          returni(cmtdoublehyphen);
782          break;
783          }
784
785  case 24:
786        
787          {
788               returnc(yylvalstr.charAt(0));
789               break;
790          }
791
792
793  case 25:
794                       
795          {
796            returni(cmpop);
797            break;
798          }
799        
800  case 26:
801           
802                  {
803
804                  if (getyysstate() == xc)
805                     {
806                      slashstar = yylvalstr.indexOf("*/");
807                        if (slashstar >= 0)
808                          {
809                              start(init);
810                              addlit(yylvalstr,slashstar+2);
811                              yylvalstr = litbufdup();
812                              yyless(slashstar+2);
813                              returni(cmtslashstar);
814                          }
815                        else
816                          {
817                              addlit(yylvalstr,1);
818                              yyless(1);
819                          }
820                     }
821                  else
822                    {
823                      nchars = yytextlen;
824                      slashstar = yylvalstr.indexOf("/*");
825                      dashdash = yylvalstr.indexOf("--");
826                      if ((slashstar >= 0) && (dashdash >= 0))
827                        {
828                          //if both appear, take the first one
829                           if (slashstar > dashdash)
830                            {slashstar = dashdash;}
831                        }
832                      else
833                        {
834                          //   if slashstar=0 then slashstar := dashdash;
835                          // add (getyysstate <> xc) to avoid something like this */--,here */ should be handled instead of --
836                          if ((slashstar >= 0) && (getyysstate() != xc)) {
837                            nchars = slashstar;
838                            }
839                        }
840
841                      while ((nchars > 1)
842                              && ( (yylvalstr.charAt(nchars-1) == '+' )
843                                  || (yylvalstr.charAt(nchars-1) =='-'))
844                              && (getyysstate() != xc))
845                        {
846                          for (ic = nchars - 1; ic>=1; ic--)
847                          {
848                            if (isopchar(yylvalstr.charAt(ic-1)))  break;
849                          }
850                          if (ic >= 1) break;
851                          nchars--;
852                        }
853
854                      if (nchars < yytextlen)
855                        {
856                          //Strip the unwanted chars from the token
857                          yyless(nchars);
858                          yylvalstr = yylvalstr.substring(0,nchars);
859                        }
860
861                          ///*
862                          // * If what we have left is only one char, and it's
863                          // * one of the characters matching "self", then
864                          // * return it as a character token the same way
865                          // * that the "self" rule would have.
866                          // * make sure @ return as self char, by james wang
867                          // */
868                          if ((nchars == 1) && (isselfchar(yylvalstr.charAt(0)) || (yylvalstr.charAt(0) == '@')))
869                            {
870                              returnc(yylvalstr.charAt(0));
871                            }
872                          else if (
873                                      (nchars >= 2)
874                                          &&(
875                                               charinarray(yylvalstr.charAt(nchars-1-1), tmparray)
876                                               && ((yylvalstr.charAt(nchars-1) == ':'))
877                                               )
878                                   )
879                              {
880                                yyless(nchars-1);
881                                 yylvalstr = yylvalstr.substring(0,nchars-1);
882                                if (nchars == 2)
883                                  returnc(yylvalstr.charAt(0));
884                                else
885                                  returni(op);
886                              }
887                            else if (
888                                        (nchars >= 2)
889                                            && (
890                                                  charinarray(yylvalstr.charAt(nchars-1-1),tmparray)
891                                                  && (yylvalstr.charAt(nchars-1) == '@')
892                                                 )
893                                       )
894                                {
895                                  yyless(nchars-1);
896                                   yylvalstr = yylvalstr.substring(0,nchars-1);
897                                  if (nchars == 2)
898                                    returnc(yylvalstr.charAt(0));
899                                  else
900                                    returni(op);
901                                }
902                            else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '~') )
903                              {
904                                      yyless(1);
905                                     returnc(yylvalstr.charAt(0));
906                              }
907                            else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '+') )
908                              {
909                                      yyless(1);
910                                     returnc(yylvalstr.charAt(0));
911                              }
912                            else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '?') )
913                              {
914                                      yyless(1);
915                                     returnc(yylvalstr.charAt(0));
916                              }
917                            else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '=')&& (yylvalstr.charAt(1) == '@') )
918                              {
919                                      yyless(1);
920                                     returnc(yylvalstr.charAt(0));
921                              }
922                            else if ( (nchars == 2) && (yylvalstr.charAt(0) == '.') && (yylvalstr.charAt(1) == '*'))
923                              {
924                                      yyless(1);
925                                     returnc(yylvalstr.charAt(0));
926                              }
927                            else if ( (nchars == 2) && (yylvalstr.charAt(0) == '+') && (yylvalstr.charAt(1) == '.'))
928                              {
929                                      yyless(1);
930                                     returnc(yylvalstr.charAt(0));
931                              }
932                            else if ( (nchars == 2) && (yylvalstr.charAt(0) == '-') && (yylvalstr.charAt(1) == '.'))
933                              {
934                                      yyless(1);
935                                     returnc(yylvalstr.charAt(0));
936                              }
937                            else if ( (nchars == 2) && (yylvalstr.charAt(1) == '~'))
938                              {
939                                      yyless(1);
940                                     returnc(yylvalstr.charAt(0));
941                              }
942                            else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '@') && (yylvalstr.charAt(1) == '@'))
943                              {
944                                      yyless(1);
945                                     returnc(yylvalstr.charAt(0));
946                              }
947                           else if ( (nchars >= 2) && (yylvalstr.charAt(0)== '|') && (yylvalstr.charAt(1) == '|'))
948                                { //||--sss ,just get || only
949                                      yyless(2);
950                                      yylvalstr = yylvalstr.substring(0,2);
951                                      returni(op);
952                                }
953                                      else if ( (nchars == 2) && (yylvalstr.charAt(1) == '?') )
954                                          {
955                                            yyless(1);
956                                            returnc(yylvalstr.charAt(0));
957                                          }
958                                      else if ( (nchars == 3) && (yylvalstr.charAt(2) == '?') )
959                                          {
960                                            //yyless(2);
961                                            //returnc(yylvalstr.charAt(0));
962                                            
963                                 yyless(nchars-1);
964                                 yylvalstr = yylvalstr.substring(0,nchars-1);
965                                if (nchars == 2)
966                                  returnc(yylvalstr.charAt(0));
967                                else
968                                  returni(op);
969                                                                            
970                                          }
971                                                                else if ( (nchars >= 3) && (yylvalstr.charAt(nchars-1) == '$' ) && (yylvalstr.charAt(nchars-2) == '$' ))
972                                                                { // =$$abc
973                                                                        if (nchars == 3){
974                                                                                yyless(1);
975                                                                                returnc(yylvalstr.charAt(0));
976                                                                        }else{
977                                                                                yyless(nchars-2);
978                                                                                yylvalstr = yylvalstr.substring(0,nchars-2);
979                                                                                returni(cmpop);
980                                                                        }
981                                                                }
982                                                                  else if (((nchars > 2) && (yylvalstr.charAt(0) == '*'))
983                                      && (yylvalstr.charAt(1) == '/')
984                                      && (getyysstate() == xc)
985                                      )
986                                   { //in comment, and find */ , then it must the end of comment
987                                     yyless(2);
988                                        addlit(yylvalstr,yytextlen);
989                                     if (xcdepth <= 0)
990                                       {
991                                         start(init);
992                                         yylvalstr = litbufdup();
993                                         returni(cmtslashstar);
994                                       }
995                                     else
996                                       xcdepth--;
997                                   }
998                              else
999                                returni(op);
1000                       }
1001
1002                  break;
1003                  }
1004
1005  case 27:
1006                        
1007          {
1008               returni(iconst);
1009               break;
1010          }
1011
1012  case 28:
1013                        
1014          {
1015            ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
1016            nchars = yytextlen;
1017            if (yylvalstr.charAt(nchars-1) == '.')
1018              {
1019                dummych1 = get_char();
1020                unget_char(dummych1);
1021                if (dummych1 == '.')
1022                  {
1023                      yyless(nchars-1);
1024                    yylvalstr = yylvalstr.substring(0,nchars - 1);
1025                    returni (iconst);
1026                    return;//exit;
1027                  }
1028              }
1029                          
1030            if (yylvalstr.startsWith(".")&&(prevToken != null)){
1031                if (prevToken.tokencode == TBaseType.ident){
1032                    yyless(1);
1033                    yylvalstr = yylvalstr.substring(0,1);
1034                    returnc(yylvalstr.charAt(0));
1035                    return;//exit;
1036                }
1037            }
1038                        
1039            returni (fconst);
1040           break;
1041          }
1042
1043  case 29:
1044                        
1045          {
1046           returni (fconst);
1047           break;
1048          }
1049
1050
1051  case 30:
1052              
1053          {
1054            int rw;
1055             
1056             nchars = yytextlen;
1057             
1058             if ((tmpDelimiter.length()>0) && (yylvalstr.endsWith(tmpDelimiter))){
1059                 yyless(nchars-tmpDelimiter.length());
1060                 yylvalstr = yylvalstr.substring(0,nchars-tmpDelimiter.length());
1061             }
1062             
1063            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1064            else if ( (yylvalstr.charAt(0) == '_') && (issparksqlCharset(yylvalstr.substring(1,nchars))) ){
1065              returni(MySQL_CHARSET_NAME);
1066            }
1067            else returni(ident);
1068            break;
1069          }
1070
1071  case 31:
1072           
1073          {
1074            returni(variable);
1075
1076            break;
1077          }
1078
1079
1080
1081  case 32:
1082         
1083          {
1084            returni(bind_v);
1085            break;
1086          }
1087                  
1088  case 33:
1089                          
1090          {
1091            returni(slash_dot);
1092            break;
1093          }
1094
1095  case 34:
1096                          
1097          {
1098            returni(mysql_null);
1099            break;
1100          }
1101                  
1102  case 35:
1103                        
1104          {
1105            returni( error);
1106            break;
1107          }
1108
1109    default:{
1110     System.out.println("fatal error in yyaction");
1111    }
1112   }//switch
1113}/*yyaction*/;
1114
1115
1116
1117        }