Source code

001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************
004   Lexical analizer for GSQLParser Java component   
005   Copyright (c) 2004-2010 by Gudu software         
006****************************************************/
007
008package gudusoft.gsqlparser;
009
010import java.util.HashMap;
011import java.io.InputStreamReader;
012
013import java.util.Locale;
014import java.io.BufferedReader;
015import java.io.IOException;
016
017        
018public class TLexerDatabricks extends TCustomLexer{
019    static int yynmarks = 0  ;
020    static int yynmatches ;
021    static int yyntrans   ;
022    static int yynstates  ;
023    static int[]  yyk,yym ; // 1 based
024    static int[]  yytint;  // 1 based
025    static TYytRec[] yyt ;  // 1 based
026    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
027    private  static  String[] keywordlist;
028    static String   table_file;
029    static HashMap<String, Integer> keywordValueList;
030    static HashMap<Integer, Integer> keywordTypeList;
031        static int[][] yystateTable;
032
033    static {
034              keywordValueList = new HashMap<String, Integer>();
035              keywordTypeList = new HashMap<Integer, Integer>();
036          table_file = "/gudusoft/gsqlparser/parser/databricks/databricks_lex_table.txt";
037                  if (TBaseType.enterprise_edition||TBaseType.databricks_edition){
038                  inittable();
039              }
040    }
041    
042    public TLexerDatabricks(){
043          super();
044          dbvendor = EDbVendor.dbvdatabricks;
045    }
046   
047public boolean canBeColumnName(int tokencode){
048    //http://blog.csdn.net/superbeck/article/details/5387476
049    boolean ret = false;
050    int modifiers = keyword_type_identifier | keyword_type_column ;
051    Integer s = keywordTypeList.get(tokencode);
052     if (s != null){
053        int modifier = s;
054        ret = (modifiers & modifier) == modifier;
055    }
056
057    return ret;
058}
059   
060 public  int iskeyword(String str){
061        int ret = -1;
062        Integer s = keywordValueList.get(str.toUpperCase(Locale.ENGLISH));
063        if( s != null){
064            ret = s;
065        }
066        return ret;// -1 means not a keyword
067     }
068
069     public int getkeywordvalue(String keyword){
070        int ret = 0;
071        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
072        if( s != null){
073            ret = s;
074         }
075        return ret;// 0 means not a keyword
076     }
077
078    public static EKeywordType getKeywordType(String keyword){
079        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
080    }    
081        
082    static void yystateLookupConfigure() {
083        int yystates = yytl.length;
084        yystateTable = new int[257][yystates];
085
086        // initialize to empty
087        for(int i = 0; i < yystates; i++) {
088            for (int j = 0; j < 257; j++)
089                yystateTable[j][i] = -1;
090        }
091
092        for(int i = 0; i < yystates; i++) {
093            int low = yytl[i];
094            int high = yyth[i];
095            for (int j = low; j <= high; j++) {
096                for (char c: yyt[j].cc) {
097                    yystateTable[c][i] = j;
098                }
099            }
100        }
101    }
102        
103    int yylex(){
104          int yyn;
105           while (true) { // top level while
106              yynew();
107              while (true){  //scan
108                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
109                     yymark(yyk[yyn]);
110                 }
111
112                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
113                    yymatch(yym[yyn]);
114                 }
115
116                 if(yytl[yystate] > yyth[yystate]){
117                     break;
118                 }
119
120                 yyscan();
121//                 yyn = yytl[yystate];
122                 totablechar();
123//                 while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
124//                   yyn++;
125//                 }
126//                 if (yyn > yyth[yystate]){
127//                     break;
128//                 }
129
130                 yyn = yystateTable[yytablechar][yystate];
131                 if (yyn == -1)
132                     break;
133                                         
134                 yystate = yyt[yyn].s;
135              } //scan
136
137              while (true){ //action
138                int yyrule;
139                if ( (yyrule = yyfind()) != -1 ){
140                   yyaction(yyrule);
141                   if (yyreject){
142                       continue;
143                   }
144                }else if( (!yydefault() ) && (yywrap()) ){
145                   yyclear();
146                   returni(0);
147                }
148                break;
149              }
150
151              if (!yydone) {
152                  continue;
153              }
154              break;
155            } // top level while
156
157           return yyretval;
158        }
159
160    static void inittable(){
161                
162                //if (yynmarks > 0) return; //init table already
163
164        String line;
165        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
166        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
167        int c=0;
168        keywordValueList.clear();
169        keywordTypeList.clear();
170        
171        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerDatabricks.class.getResourceAsStream(table_file)));
172
173            try{
174                while( (line = br.readLine()) != null){
175                          if (line.trim().startsWith("yynmarks=")){
176                             String[] ss = line.split("[=;]");
177                              yynmarks=Integer.parseInt(ss[1].trim());
178                              yyk = new int[yynmarks+1];
179                          }else if (line.trim().startsWith("yynmatches=")){
180                              String[] ss = line.split("[=;]");
181                               yynmatches=Integer.parseInt(ss[1].trim());
182                               yym = new int[yynmatches+1];
183                          }else if (line.trim().startsWith("yyntrans=")){
184                              String[] ss = line.split("[=;]");
185                               yyntrans=Integer.parseInt(ss[1].trim());
186                               yytint = new int[yyntrans+1];
187                               yyt = new TYytRec[yyntrans+1];
188                          }else if (line.trim().startsWith("yynstates=")){
189                              String[] ss = line.split("[=;]");
190                               yynstates=Integer.parseInt(ss[1].trim());
191                               yykl = new int[yynstates];
192                               yykh = new int[yynstates];
193                              yyml = new int[yynstates];
194                              yymh = new int[yynstates];
195                              yytl = new int[yynstates];
196                              yyth = new int[yynstates];
197                          }else if (line.trim().startsWith("<end>")){
198                              if (inyyk){
199                                  inyyk = false;
200                                 if (yynmarks+1 != yyk_count ){
201                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
202                                 }
203                              }
204                              else if(inyym){
205                                     inyym = false;
206                                    if (yynmatches+1 != yym_count ){
207                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
208                                    }
209                              }
210                              else if(inyykl){
211                                     inyykl = false;
212                                    if (yynstates != yykl_count ){
213                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
214                                    }
215                              }
216                              else if(inyykh){
217                                     inyykh = false;
218                                    if (yynstates != yykh_count ){
219                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
220                                    }
221                              }
222                              else if(inyyml){
223                                     inyyml = false;
224                                    if (yynstates != yyml_count ){
225                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
226                                    }
227                              }
228                              else if(inyymh){
229                                     inyymh = false;
230                                    if (yynstates != yymh_count ){
231                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
232                                    }
233                              }
234                              else if(inyytl){
235                                     inyytl = false;
236                                    if (yynstates != yytl_count ){
237                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
238                                    }
239                              }
240                              else if(inyyth){
241                                     inyyth = false;
242                                    if (yynstates != yyth_count ){
243                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
244                                    }
245                              }
246                              else if(inyytint){
247                                     inyytint = false;
248                                    if (yyntrans + 1 != yytint_count ){
249                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
250                                    }
251                              }
252                              else if(inyyt){
253                                     inyyt = false;
254                                    if (yyntrans+1 != yyt_count ){
255                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
256                                    }
257                              }
258                              else if(inkeyword){
259                                     inkeyword = false;
260                              }
261                          }else if(line.trim().startsWith("yyk =")){
262                             inyyk = true; 
263                          }else if(line.trim().startsWith("yym =")){
264                             inyym = true;
265                          }else if(line.trim().startsWith("yykl =")){
266                             inyykl = true;
267                          }else if(line.trim().startsWith("yykh =")){
268                             inyykh = true;
269                          }else if(line.trim().startsWith("yyml =")){
270                             inyyml = true;
271                          }else if(line.trim().startsWith("yymh =")){
272                             inyymh = true;
273                          }else if(line.trim().startsWith("yytl =")){
274                             inyytl = true;
275                          }else if(line.trim().startsWith("yyth =")){
276                             inyyth = true;
277                          }else if(line.trim().startsWith("yytint =")){
278                             inyytint = true;
279                          }else if(line.trim().startsWith("yyt =")){
280                             inyyt = true;
281                        }else if(line.trim().startsWith("keywordsvalue =")){
282                           inkeyword = true;
283                        }else if(inyyk){
284                             String[] ss = line.split("[,]");
285                               for(int j=0;j<ss.length;j++){
286                                   // System.out.println(ss[j].trim());
287                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
288                               }
289                          }else if(inyym){
290                               String[] ss = line.split("[,]");
291                                 for(int j=0;j<ss.length;j++){
292                                     // System.out.println(ss[j].trim());
293                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
294                                 }
295                          }else if(inyykl){
296                               String[] ss = line.split("[,]");
297                                 for(int j=0;j<ss.length;j++){
298                                    //  System.out.println(ss[j].trim());
299                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
300                                 }
301                          }else if(inyykh){
302                               String[] ss = line.split("[,]");
303                                 for(int j=0;j<ss.length;j++){
304                                     // System.out.println(ss[j].trim());
305                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
306                                 }
307                          }else if(inyyml){
308                               String[] ss = line.split("[,]");
309                                 for(int j=0;j<ss.length;j++){
310                                     // System.out.println(ss[j].trim());
311                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
312                                 }
313                          }else if(inyymh){
314                               String[] ss = line.split("[,]");
315                                 for(int j=0;j<ss.length;j++){
316                                     // System.out.println(ss[j].trim());
317                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
318                                 }
319                          }else if(inyytl){
320                               String[] ss = line.split("[,]");
321                                 for(int j=0;j<ss.length;j++){
322                                     // System.out.println(ss[j].trim());
323                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
324                                 }
325                          }else if(inyyth){
326                               String[] ss = line.split("[,]");
327                                 for(int j=0;j<ss.length;j++){
328                                     // System.out.println(ss[j].trim());
329                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
330                                 }
331                          }else if(inyytint){
332                               String[] ss = line.split("[,]");
333                                 for(int j=0;j<ss.length;j++){
334                                     // System.out.println(ss[j].trim());
335                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
336                                 }
337                          }else if(inyyt){
338                                //System.out.println(line.trim());
339
340                              c = 0;
341                              String[] st = line.trim().split(",,");
342                              char[] tmp = new char[st.length];
343                              for(int i=0;i<st.length;i++){
344
345                                  if(st[i].startsWith("\'")) {
346                                      if(st[i].length() == 3){  // 'a'
347                                          tmp[c++] = st[i].charAt(1);
348                                      }else if(st[i].length() == 4) { // '\\'
349                                          tmp[c++] = st[i].charAt(2);
350                                      }else{
351                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
352                                      }
353                                  }else{
354                                      try{
355                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
356                                          } catch (NumberFormatException nfe) {
357                                             System.out.println("NumberFormatException: " + nfe.getMessage());
358                                          }
359                                  }
360                              } //while hasmoreTokens
361
362                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
363                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
364                              yyt_count++;
365
366                          }else if(inkeyword){
367                              String[] ss =line.split("[=]");
368
369                              int val1 = -1;
370                              int val2 = -1;
371                              try {
372                                  val1 = Integer.parseInt(ss[1]);
373                                  val2 = Integer.parseInt(ss[2]);
374                              }
375                              catch (NumberFormatException nfe) {
376                                  System.out.println("NumberFormatException: " + nfe.getMessage());
377                              }
378                              keywordValueList.put(ss[0].toUpperCase(),val1);
379                              keywordTypeList.put(val1,val2);
380                             }
381                }
382                }catch(IOException e){
383                  System.out.println(e.toString());
384                }
385                                
386                                yystateLookupConfigure();
387
388    }
389
390
391    void yyaction(int yyruleno){
392                                
393
394      int ic;
395      char[] tmparray = {'=','+','-','*','/','>','<'};
396
397      yylvalstr = getyytext();
398  /* actions: */
399  switch(yyruleno){
400  case 1:
401                   
402                {
403                                        if (yylvalstr.equalsIgnoreCase(dolqstart))
404                                        {
405                                                //dolqstart = "";
406            start(init);
407            addlit(yylvalstr,yytextlen);
408            yylvalstr = litbufdup();
409                                                returni(sconst);
410                                        }
411                                        else
412                                        {
413            //nchars = yytextlen;
414            addlit(yylvalstr, yytextlen-1);
415            yyless(yytextlen-1);
416            return;
417                                        }
418                                        //System.out.println("<xdolq>{dolqdelim}: "+dolqstart);
419                                        break;
420                }
421                                
422  case 2:
423            
424          {
425              start(xdolq);
426              startlit();
427              dolqstart = yylvalstr;
428              addlit(yylvalstr,yytextlen);
429              //System.out.println("dolqdelim: "+dolqstart);
430              break;
431          }
432
433  case 3:
434                    
435        {
436                addlit(yylvalstr, yytextlen);
437          //System.out.println("<xdolq>{dolqinside}: "+yylvalstr);
438                break;
439        }
440                                
441  case 4:
442                    
443        {
444                        addlit(yylvalstr, yytextlen);
445                        break;
446        }
447                                
448  case 5:
449                        
450                {
451         addlitchar(yylvalstr.charAt(0));
452         break;
453                }
454          
455
456  case 6:
457           
458          {
459              if (getyysstate() == xq)
460              {
461                  nchars = yytextlen;
462                  addlit(yylvalstr, yytextlen-1);
463                  yyless(nchars-1);
464                  return;//exit;
465              }
466
467              start(xq);
468              startlit();
469              addlit(yylvalstr,yytextlen);
470              break;
471          }
472
473  case 7:
474                
475          {
476              if (getyysstate() == xq)
477              {
478                  nchars = yytextlen;
479                  addlit(yylvalstr, yytextlen-1);
480                  yyless(nchars-1);
481                  return;//exit;
482              }
483
484              start(xq);
485              startlit();
486              addlit(yylvalstr,yytextlen);
487              break;
488          }
489
490  case 8:
491                
492          {
493              if (getyysstate() == xq)
494              {
495                  nchars = yytextlen;
496                  addlit(yylvalstr, yytextlen-1);
497                  yyless(nchars-1);
498                  return;//exit;
499              }
500
501              start(xq);
502              startlit();
503              addlit(yylvalstr,yytextlen);
504              break;
505          }
506
507  case 9:
508                
509        {
510                    addlit(yylvalstr,yytextlen);
511                    if (xcdepth <= 0)
512                       {
513                        start(init);
514                        yylvalstr = litbufdup();
515                        returni(cmtslashstar);
516                       }
517                    else
518                       xcdepth--;
519
520                  break;
521        }
522
523
524  case 10:
525                
526                  {
527                      if (yylvalstr.equalsIgnoreCase("/*+")){
528                         xcdepth++;
529                          yyless(2);
530                          addlit(yylvalstr,yytextlen);
531                      }else{
532                          yyless(1);
533                          addlit(yylvalstr,1);
534                      }
535        
536                      break;
537
538                   }
539
540  case 11:
541                        
542                  {
543
544                      if (getyysstate() == xq)
545                      {
546                          nchars = yytextlen;
547                          addlit(yylvalstr, yytextlen-1);
548                          yyless(nchars-1);
549                          return;//exit;
550                      }
551
552                      xcdepth = 0;
553                      start(xc);
554                      startlit();
555                      yyless(2);
556                      addlit(yylvalstr,yytextlen);
557
558                  break;
559                  }
560
561  case 12:
562                
563                  {
564                    addlit(yylvalstr,yytextlen);
565        
566                    break;
567                  }
568
569  case 13:
570                
571                  {
572                      addlitchar(yylvalstr.charAt(0));
573        
574                      break;
575                  }
576                                  
577  case 14:
578                       
579        {
580                start(init);
581                addlit(yylvalstr, yytextlen);
582                yylvalstr = litbufdup();
583                returni(ident);
584                break;
585        }
586
587  case 15:
588                        
589        {
590          if ((getyysstate() == xd)||(getyysstate() == xq))
591          {
592              addlit(yylvalstr, 1);
593              yyless(1);
594              return;//exit;
595          }     
596                                        //token_start := yylvalStr;
597                                        start(xbacktick);
598                                        startlit();
599                                        addlit(yylvalstr, yytextlen);
600                break;
601        }
602
603
604  case 16:
605                       
606        {
607                addlit(yylvalstr, yytextlen);
608                break;
609  }
610
611  case 17:
612                       
613        {
614    addlit(yylvalstr, yytextlen);
615                break;
616   }                              
617
618  case 18:
619                
620                  {
621                      start(init);
622                      addlit(yylvalstr, yytextlen);
623                      yylvalstr = litbufdup();
624                      if( yylvalstr.startsWith("b")|| (yylvalstr.startsWith("B"))){
625                          returni(bconst);
626                      }else if( yylvalstr.startsWith("x")|| (yylvalstr.startsWith("X"))){
627                          returni(xconst);
628                      }else 
629                      {
630                      returni(sconst);
631                      }
632                      break;
633                  }
634
635  case 19:
636                
637                    {
638                    if (insqlpluscmd){
639                        yyless(0);
640                        yylvalstr = litbufdup();
641                        start(init);
642                        returni(sconst);
643                    }else{
644                      addlit(yylvalstr,yytextlen);
645                    }
646
647                    break;
648                }
649  case 20:
650            
651                  {
652                      if (getyysstate() == xq)
653                      {
654                          nchars = yytextlen;
655                          addlit(yylvalstr, yytextlen-1);
656                          yyless(nchars-1);
657                          return;//exit;
658                      }
659
660                      start(xq);
661                      startlit();
662                      addlit(yylvalstr,yytextlen);
663
664                      dummych1 = get_char();
665                      if (dummych1 == '\\') // recognize string like '\'
666                        {
667                          dummych2 = get_char();
668                          if (dummych2 == '\'')
669                          {
670                             // start(init);
671                              addlit("\\", 1);
672                              addlit("\'", 1);
673                              //yylvalstr = litbufdup();
674                              //returni(sconst);
675                          }
676                          else
677                           {
678                              unget_char(dummych2);
679                              unget_char(dummych1);
680                            }
681                        }
682                      else
683                      { unget_char(dummych1);}
684
685                      break;
686                  }
687
688  case 21:
689                        
690                  {
691                          start(xq);
692                          startlit();
693                          addlit(yylvalstr, yytextlen);
694                          dummych1 = get_char();
695                          if (dummych1 == '\\') // recognize string like '\'
696                            {
697                              dummych2 = get_char();
698                              if (dummych2 == '\'')
699                              {
700                                 // start(init);
701                                  addlit("\\", 1);
702                                  addlit("\'", 1);
703                                  //yylvalstr = litbufdup();
704                                  //returni(sconst);
705                              }
706                              else
707                               {
708                                  unget_char(dummych2);
709                                  unget_char(dummych1);
710                                }
711                            }
712                          else
713                          { unget_char(dummych1);}
714
715                      break;
716                  }
717
718  case 22:
719                
720                  {
721                      addlit(yylvalstr, yytextlen);
722                      break;
723                  }
724
725  case 23:
726                
727          {
728            dummych1 = get_char();
729            unget_char(dummych1);
730            if (dummych1 == (char)10)
731              {
732                if (insqlpluscmd){
733                  nchars = yytextlen;
734                  if(yylvalstr.charAt(nchars-1) == (char)13){
735                      yyless(nchars - 1);
736                      yylvalstr = yylvalstr.substring(0,nchars);
737                  }
738                    start(init);
739                    addlit(yylvalstr, nchars-1);
740                    yylvalstr = litbufdup();
741                    returni(sconst); //in sqlplus command, characters between ' and return is treated as a string
742
743                }else{
744                        dummych1 = get_char();
745                        addlit(yylvalstr+dummych1, yytextlen+1);
746                }
747              } else if (dummych1 == '\\')
748                  {
749                      dummych1 = get_char();
750                      dummych2 = get_char();
751                      addlit(yylvalstr+dummych1+dummych2, yytextlen+2);
752                      //addlit(yylvalstr, yytextlen);
753                  }
754                else
755                    { addlit(yylvalstr, yytextlen);}
756
757                        break;
758         }
759 
760  case 24:
761                 
762                  {
763                      addlit(yylvalstr, yytextlen);
764                      break;
765                  }
766
767
768
769  case 25:
770                
771                  {
772                      start(init);
773                      addlit(yylvalstr, yytextlen);
774                      if ((literallen == 0) && (!insqlpluscmd))
775                        {returni (error);}
776                      if (literallen >= namedatalen)
777                      {
778                         setlengthofliteralbuf(namedatalen);
779                         literallen = namedatalen;
780                      }
781                      yylvalstr = litbufdup();
782                      returni (ident);
783
784                      break;
785                  }
786
787
788  case 26:
789                
790                  {
791                      if (insqlpluscmd){
792                          yyless(0);
793                          yylvalstr = litbufdup();
794                          start(init);
795                          returni(sconst);
796                      }else{
797                          addlit(yylvalstr, yytextlen);
798                      }
799
800                      break;
801                  }
802
803  case 27:
804               
805                  {
806                      addlit(yylvalstr, yytextlen);
807                      break;
808                  }
809
810  case 28:
811                        
812                  {
813                      start(xd);
814                      startlit();
815                      addlit(yylvalstr, yytextlen);
816                      break;
817                  }
818
819  case 29:
820                
821                  {
822                    dummych1 = get_char();
823                    unget_char(dummych1);
824                    if (dummych1 == (char)10)
825                      {
826                          if (insqlpluscmd){
827                            nchars = yytextlen;
828                            if(yylvalstr.charAt(nchars-1) == (char)13){
829                                yyless(nchars - 1);
830                                yylvalstr = yylvalstr.substring(0,nchars);
831                            }
832                              start(init);
833                              addlit(yylvalstr, nchars-1);
834                              yylvalstr = litbufdup();
835                              returni(sconst); //in sqlplus command, characters between ' and return is treated as a string
836
837                          }else{
838                          dummych1 = get_char();
839                          addlit(yylvalstr+dummych1, yytextlen+1);
840                          }
841
842                      } else
843                        addlit(yylvalstr, yytextlen);
844
845                  break;
846                  }
847
848  case 30:
849                
850                  {
851                    returni(lexnewline);
852                    break;
853                  }
854
855  case 31:
856        
857                  {
858                      returni(lexspace);
859                      break;
860                  }
861
862  case 32:
863                
864                  {
865                  if ((getyysstate() == xq)
866                      || (getyysstate() == xd)
867                      || (getyysstate() == xc)
868                                          || (getyysstate() == xbacktick)
869                   )
870                  {
871                      addlit(yylvalstr, 1);
872                      yyless(1);
873                      return;//exit;
874                  }
875
876                  returni(cmtdoublehyphen);
877                  break;
878                  }
879
880  case 33:
881        
882                  {
883                       returnc(yylvalstr.charAt(0));
884                       break;
885                  }
886
887  case 34:
888                       
889                  {
890                    returni(cmpop);
891                    break;
892                  }
893
894  case 35:
895           
896                  {
897
898                  if (getyysstate() == xc)
899                     {
900                      slashstar = yylvalstr.indexOf("*/");
901                        if (slashstar >= 0)
902                          {
903                              start(init);
904                              addlit(yylvalstr,slashstar+2);
905                              yylvalstr = litbufdup();
906                              yyless(slashstar+2);
907                              returni(cmtslashstar);
908                          }
909                        else
910                          {
911                              addlit(yylvalstr,1);
912                              yyless(1);
913                          }
914                     }
915                  else if (getyysstate() == xq)
916                  {
917                              addlit(yylvalstr,1);
918                              yyless(1);
919                  }
920                  else
921                    {
922                      nchars = yytextlen;
923                      slashstar = yylvalstr.indexOf("/*");
924                      dashdash = yylvalstr.indexOf("--");
925                      if ((slashstar > 0) && (dashdash > 0))
926                        {
927                          //if both appear, take the first one
928                           if (slashstar > dashdash)
929                            {slashstar = dashdash;}
930                        }
931                      else
932                        {
933                          //   if slashstar=0 then slashstar := dashdash;
934                          // add (getyysstate <> xc) to avoid something like this */--,here */ should be handled instead of --
935                          if ((slashstar > 0) && (getyysstate() != xc)) {
936                            nchars = slashstar;
937                            }
938                        }
939
940                      while ((nchars > 1)
941                              && ( (yylvalstr.charAt(nchars-1) == '+' )
942                                  || (yylvalstr.charAt(nchars-1) =='-'))
943                              && (getyysstate() != xc))
944                        {
945                          for (ic = nchars - 1; ic>=1; ic--)
946                          {
947                            if (isopchar(yylvalstr.charAt(ic-1)))  break;
948                          }
949                          if (ic >= 1) break;
950                          nchars--;
951                        }
952
953                      if (nchars < yytextlen)
954                        {
955                          //Strip the unwanted chars from the token
956                          yyless(nchars);
957                          yylvalstr = yylvalstr.substring(0,nchars);
958                        }
959
960                          ///*
961                          // * If what we have left is only one char, and it's
962                          // * one of the characters matching "self", then
963                          // * return it as a character token the same way
964                          // * that the "self" rule would have.
965                          // * make sure @ return as self char, by james wang
966                          // */
967                          if ((nchars == 1) && (isselfchar(yylvalstr.charAt(0)) || (yylvalstr.charAt(0) == '@')))
968                            {
969                              returnc(yylvalstr.charAt(0));
970                            }
971                          else if (
972                                      (nchars >= 2)
973                                          &&(
974                                               charinarray(yylvalstr.charAt(nchars-1-1), tmparray)
975                                               && ((yylvalstr.charAt(nchars-1) == ':')
976                                                                ||(yylvalstr.charAt(nchars-1) == '.')
977                                                        )
978                                               )
979                                   )
980                              {
981                                yyless(nchars-1);
982                                 yylvalstr = yylvalstr.substring(0,nchars-1);
983                                if (nchars == 2)
984                                  returnc(yylvalstr.charAt(0));
985                                else
986                                  returni(op);
987                              }
988                            else if (
989                                        (nchars >= 2)
990                                            && (
991                                                  charinarray(yylvalstr.charAt(nchars-1-1),tmparray)
992                                                  && (yylvalstr.charAt(nchars-1) == '&')
993                                                 )
994                                       )
995                                {
996                                  yyless(nchars-1);
997                                   yylvalstr = yylvalstr.substring(0,nchars-1);
998                                  if (nchars == 2)
999                                    returnc(yylvalstr.charAt(0));
1000                                  else
1001                                    returni(op);
1002                                }
1003                            else if ( (nchars == 2) && (yylvalstr.charAt(0) == '.') && (yylvalstr.charAt(1) == '*'))
1004                              {
1005                                      yyless(1);
1006                                     returnc(yylvalstr.charAt(0));
1007                              }
1008                                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '?')) ))
1009                                          {
1010                                            yyless(1);
1011                                            returnc(yylvalstr.charAt(0));
1012                                          }
1013                                      else if ((nchars >= 2) && ((yylvalstr.charAt(0) == '>') && ( (yylvalstr.charAt(1) == '>')) ))
1014                                          {
1015                                            yyless(1);
1016                                            returnc(yylvalstr.charAt(0));
1017                                          }
1018                                      else if ((nchars >= 2) && ((yylvalstr.charAt(0) == '<') && ( (yylvalstr.charAt(1) == '<')) ))
1019                                          {
1020                                            yyless(1);
1021                                            returnc(yylvalstr.charAt(0));
1022                                          }
1023                          else if ( (nchars >= 2) && ((yylvalstr.charAt(0) == '@')&&(yylvalstr.charAt(1) != '>')))
1024                            {
1025                                    yyless(1);
1026                                   returnc(yylvalstr.charAt(0));
1027                            }
1028                          else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '?')&&(yylvalstr.charAt(1) != '&')&&(yylvalstr.charAt(1) != '|'))
1029                            {
1030                                    yyless(1);
1031                                   returnc(yylvalstr.charAt(0));
1032                            }
1033                              else if (((nchars > 2) && (yylvalstr.charAt(0) == '*'))
1034                                      && (yylvalstr.charAt(1) == '/')
1035                                      && (getyysstate() == xc)
1036                                      )
1037                                   { //in comment, and find */ , then it must the end of comment
1038                                     yyless(2);
1039                                        addlit(yylvalstr,yytextlen);
1040                                     if (xcdepth <= 0)
1041                                       {
1042                                         start(init);
1043                                         yylvalstr = litbufdup();
1044                                         returni(cmtslashstar);
1045                                       }
1046                                     else
1047                                       xcdepth--;
1048                                   }
1049                              else
1050                                returni(op);
1051                       }
1052
1053                  break;
1054                  }
1055
1056  case 36:
1057                        
1058                  {
1059                       returni(iconst);
1060                       break;
1061                  }
1062
1063  case 37:
1064                        
1065                  {
1066                    ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
1067                    nchars = yytextlen;
1068                    if (yylvalstr.charAt(nchars-1) == '.')
1069                      {
1070                        dummych1 = get_char();
1071                        unget_char(dummych1);
1072                        if (dummych1 == '.')
1073                          {
1074                              yyless(nchars-1);
1075                            yylvalstr = yylvalstr.substring(0,nchars - 1);
1076                            returni (iconst);
1077                            return;//exit;
1078                          }
1079                      }
1080                    returni (fconst);
1081                   break;
1082                  }
1083
1084  case 38:
1085                        
1086                  {
1087                   returni (fconst);
1088                   break;
1089                  }
1090
1091  case 39:
1092              
1093                  {
1094                     boolean dollarConstant = false;
1095                          if (getyysstate() == xdolq){
1096                           int p = yylvalstr.indexOf("$");
1097                       if (p > 0){
1098                           dollarConstant = true;
1099                           addlit(yylvalstr, p);
1100                          yyless(p);
1101                          return;
1102                       }
1103                      }
1104
1105                          if (!dollarConstant){
1106                            int rw;
1107                            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1108                            else
1109                                {returni(ident);}
1110                          }
1111                      break;
1112                  }
1113 
1114  case 40:
1115        
1116      {
1117        if (getyysstate() == xdolq){
1118                                        addlit(yylvalstr, yytextlen);
1119          return;
1120        }else{
1121                returni (param);
1122        }
1123        
1124       break;
1125      }
1126
1127
1128  case 41:
1129           
1130        {
1131     returni (typecast);
1132     break;
1133        }
1134        
1135  case 42:
1136           
1137        {
1138     returni (double_dot);
1139     break;
1140        }
1141
1142  case 43:
1143               
1144        {
1145     returni (assign_sign);
1146     break;
1147        }
1148
1149  case 44:
1150          
1151        {
1152                returni(ident);
1153                        break;
1154        }
1155        
1156  case 45:
1157            
1158        {
1159                returni(atversion);
1160                        break;
1161        }
1162
1163                                  
1164  case 46:
1165                        
1166                  {
1167                   returni (error);
1168                   break;
1169                  }
1170
1171    default:{
1172     System.out.println("fatal error in yyaction");
1173    }
1174   }//switch
1175}/*yyaction*/;
1176
1177
1178
1179        }