Source code

001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************}
004{   Lexical analizer for GSQLParser component java version    }
005{   Copyright (c) 2004-2025 by Gudu software              }
006{****************************************************/
007
008package gudusoft.gsqlparser;
009
010
011import java.util.*;
012import java.io.InputStreamReader;
013
014import java.io.BufferedReader;
015import java.io.IOException;
016
017import gudusoft.gsqlparser.stmt.teradata.utilities.*;
018
019public class TLexerTeradata extends TCustomLexer{
020    static int yynmarks = 0  ;
021    static int yynmatches ;
022    static int yyntrans   ;
023    static int yynstates  ;
024    static int[]  yyk,yym ; // 1 based
025    static int[]  yytint;  // 1 based
026    static TYytRec[] yyt ;  // 1 based
027    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
028    private  static  String[] keywordlist;
029    static String   table_file;
030    static HashMap<String, Integer> keywordValueList;
031    static HashMap<Integer, Integer> keywordTypeList;
032          static String  bteqCmdStr,multiLoadCmdStr,fastExportCmdStr,fastLoadCmdStr;
033    static List<String> bteqCmdList,multiLoadCmdList,fastExportCmdList,fastLoadCmdList;
034        static int[][] yystateTable;
035
036    static {
037                        table_file = "/gudusoft/gsqlparser/parser/teradata/teradata_lex_table.txt";
038      keywordValueList = new HashMap<String, Integer>();
039              keywordTypeList = new HashMap<Integer, Integer>();
040                        bteqCmdList = new ArrayList<String>( );
041                        multiLoadCmdList = new ArrayList<String>( );
042                        fastExportCmdList = new ArrayList<String>( );
043                        fastLoadCmdList = new ArrayList<String>( );
044                        if (TBaseType.enterprise_edition||TBaseType.teradata_edition){
045                                inittable();
046                        }
047    }
048
049    public TLexerTeradata(){
050          super();
051          dbvendor = EDbVendor.dbvteradata;
052          sourcetokens = new TSourceTokenList();
053    }
054
055    public void setTeradataUtilityType(TeradataUtilityType teradataUtilityType) {
056        this.teradataUtilityType = teradataUtilityType;
057    }
058
059    private TeradataUtilityType teradataUtilityType = TeradataUtilityType.BTEQ;
060    
061    public TSourceTokenList sourcetokens;
062
063
064
065    int issystemvariable(String str){
066        return -1;// -1 means not a system variable
067    }
068
069public boolean canBeColumnName(int tokencode){
070    //http://blog.csdn.net/superbeck/article/details/5387476
071    boolean ret = false;
072    int modifiers = keyword_type_identifier | keyword_type_column ;
073    Integer s = keywordTypeList.get(tokencode);
074     if (s != null){
075        int modifier = s;
076        ret = (modifiers & modifier) == modifier;
077    }
078
079    return ret;
080}
081
082   public EFindSqlStateType cmdType(TSourceToken token){
083        
084        // use  isMultiLoadCmd,isFastExportCmd,isFastLoadCmd,isBteqCmd to find the command type,
085        String str = token.toString();
086        String str2 = null;
087        String str3 = null;
088        if (token.nextSolidToken() != null) {
089            str2 = token.nextSolidToken().toString();
090            if (token.nextSolidToken().nextSolidToken() != null) {
091              str3 = token.nextSolidToken().nextSolidToken().toString();
092            }
093        }
094
095    // First check the longest possible command (str + str2 + str3)
096    if (str3 != null) {
097        String combinedStr3 = str + " " + str2 + " " + str3;
098        int matchCount = 0;
099        boolean isMultiLoad3 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr3) != null;
100        boolean isFastExport3 = FastExportCmdType.searchFastExportCmd(combinedStr3) != null;
101        boolean isFastLoad3 = FastLoadCmdType.searchFastLoadCmd(combinedStr3) != null;
102        boolean isBteq3 = BteqCmdType.searchBteqCmd(combinedStr3) != null;
103
104        if (isMultiLoad3) matchCount++;
105        if (isFastExport3) matchCount++;
106        if (isFastLoad3) matchCount++;
107        if (isBteq3) matchCount++;
108
109        if (matchCount > 1) {
110            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq3) return EFindSqlStateType.stBTEQCmd;
111            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
112            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport3) return EFindSqlStateType.stFastExportCmd;
113            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
114
115            // throw new IllegalStateException("Ambiguous command '" + combinedStr3 + "' matches multiple utility types");
116            matchCount = 1;
117        }
118
119        if (matchCount == 1) {
120            if (isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
121            if (isFastExport3) return EFindSqlStateType.stFastExportCmd;
122            if (isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
123            if (isBteq3) return EFindSqlStateType.stBTEQCmd;
124        }
125    }
126
127    // Then check medium length command (str + str2)
128    if (str2 != null) {
129        String combinedStr2 = str + " " + str2;
130        int matchCount = 0;
131        boolean isMultiLoad2 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr2) != null;
132        boolean isFastExport2 = FastExportCmdType.searchFastExportCmd(combinedStr2) != null;
133        boolean isFastLoad2 = FastLoadCmdType.searchFastLoadCmd(combinedStr2) != null;
134        boolean isBteq2 = BteqCmdType.searchBteqCmd(combinedStr2) != null;
135
136        if (isMultiLoad2) matchCount++;
137        if (isFastExport2) matchCount++;
138        if (isFastLoad2) matchCount++;
139        if (isBteq2) matchCount++;
140
141        if (matchCount > 1) {
142            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq2) return EFindSqlStateType.stBTEQCmd;
143            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
144            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport2) return EFindSqlStateType.stFastExportCmd;
145            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
146
147            //throw new IllegalStateException("Ambiguous command '" + combinedStr2 + "' matches multiple utility types");
148            matchCount = 1;
149        }
150        if (matchCount == 1) {
151            if (isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
152            if (isFastExport2) return EFindSqlStateType.stFastExportCmd;
153            if (isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
154            if (isBteq2) return EFindSqlStateType.stBTEQCmd;
155        }
156    }
157
158    // Finally check single word command (str)
159    boolean isMultiLoad = MultiLoadCmdType.searchMultiLoadCmd(str) != null;
160    boolean isFastExport = FastExportCmdType.searchFastExportCmd(str) != null;
161    boolean isFastLoad = FastLoadCmdType.searchFastLoadCmd(str) != null;
162    boolean isBteq = BteqCmdType.searchBteqCmd(str) != null;
163    int matchCount = 0;
164    if (isMultiLoad) matchCount++;
165    if (isFastExport) matchCount++;
166    if (isFastLoad) matchCount++;
167    if (isBteq) matchCount++;
168
169    if (matchCount > 1) {
170        if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq) return EFindSqlStateType.stBTEQCmd;
171        if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
172        if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport) return EFindSqlStateType.stFastExportCmd;
173        if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad) return EFindSqlStateType.stFastLoadCmd;
174
175       // throw new IllegalStateException("Ambiguous command '" + str + "' matches multiple utility types");
176        matchCount = 1;
177    }
178
179    if (matchCount == 1) {
180      if (isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
181      if (isFastExport) return EFindSqlStateType.stFastExportCmd;
182      if (isFastLoad) return EFindSqlStateType.stFastLoadCmd;
183      if (isBteq) return EFindSqlStateType.stBTEQCmd;
184  }
185    return EFindSqlStateType.stnormal;
186
187  }
188
189
190
191
192 public  int iskeyword(String str){
193        int ret = -1;
194        Integer s = keywordValueList.get(str.toUpperCase(Locale.ENGLISH));
195        if( s != null){
196            ret = s;
197        }
198        return ret;// -1 means not a keyword
199     }
200         
201    public static EKeywordType getKeywordType(String keyword){
202        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
203    }            
204
205     public int getkeywordvalue(String keyword){
206        int ret = 0;
207        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
208        if( s != null){
209            ret = s;
210         }
211        return ret;// 0 means not a keyword
212     }
213
214    static void yystateLookupConfigure() {
215        int yystates = yytl.length;
216        yystateTable = new int[257][yystates];
217
218        // initialize to empty
219        for(int i = 0; i < yystates; i++) {
220            for (int j = 0; j < 257; j++)
221                yystateTable[j][i] = -1;
222        }
223
224        for(int i = 0; i < yystates; i++) {
225            int low = yytl[i];
226            int high = yyth[i];
227            for (int j = low; j <= high; j++) {
228                for (char c: yyt[j].cc) {
229                    yystateTable[c][i] = j;
230                }
231            }
232        }
233    }
234        
235    int yylex(){
236          int yyn;
237           while (true) { // top level while
238              yynew();
239              while (true){  //scan
240                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
241                     yymark(yyk[yyn]);
242                 }
243
244                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
245                    yymatch(yym[yyn]);
246                 }
247
248                 if(yytl[yystate] > yyth[yystate]){
249                     break;
250                 }
251
252                 yyscan();
253//                 yyn = yytl[yystate];
254                 totablechar();
255//                 while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
256//                   yyn++;
257//                 }
258//                 if (yyn > yyth[yystate]){
259//                     break;
260//                 }
261
262                 yyn = yystateTable[yytablechar][yystate];
263                 if (yyn == -1)
264                     break;
265                                         
266                 yystate = yyt[yyn].s;
267              } //scan
268
269              while (true){ //action
270                int yyrule;
271                if ( (yyrule = yyfind()) != -1 ){
272                   yyaction(yyrule);
273                   if (yyreject){
274                       continue;
275                   }
276                }else if( (!yydefault() ) && (yywrap()) ){
277                   yyclear();
278                   returni(0);
279                }
280                break;
281              }
282
283              if (!yydone) {
284                  continue;
285              }
286              break;
287            } // top level while
288
289           return yyretval;
290        }
291
292    
293
294    static void inittable(){
295                
296                //if (yynmarks > 0) return; //init table already
297
298        String line;
299        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
300        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
301        int c=0;
302        keywordValueList.clear();
303        keywordTypeList.clear();
304        
305        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerTeradata.class.getResourceAsStream(table_file)));
306
307            try{
308                while( (line = br.readLine()) != null){
309                          if (line.trim().startsWith("yynmarks=")){
310                             String[] ss = line.split("[=;]");
311                              yynmarks=Integer.parseInt(ss[1].trim());
312                              yyk = new int[yynmarks+1];
313                          }else if (line.trim().startsWith("yynmatches=")){
314                              String[] ss = line.split("[=;]");
315                               yynmatches=Integer.parseInt(ss[1].trim());
316                               yym = new int[yynmatches+1];
317                          }else if (line.trim().startsWith("yyntrans=")){
318                              String[] ss = line.split("[=;]");
319                               yyntrans=Integer.parseInt(ss[1].trim());
320                               yytint = new int[yyntrans+1];
321                               yyt = new TYytRec[yyntrans+1];
322                          }else if (line.trim().startsWith("yynstates=")){
323                              String[] ss = line.split("[=;]");
324                               yynstates=Integer.parseInt(ss[1].trim());
325                               yykl = new int[yynstates];
326                               yykh = new int[yynstates];
327                              yyml = new int[yynstates];
328                              yymh = new int[yynstates];
329                              yytl = new int[yynstates];
330                              yyth = new int[yynstates];
331                          }else if (line.trim().startsWith("<end>")){
332                              if (inyyk){
333                                  inyyk = false;
334                                 if (yynmarks+1 != yyk_count ){
335                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
336                                 }
337                              }
338                              else if(inyym){
339                                     inyym = false;
340                                    if (yynmatches+1 != yym_count ){
341                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
342                                    }
343                              }
344                              else if(inyykl){
345                                     inyykl = false;
346                                    if (yynstates != yykl_count ){
347                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
348                                    }
349                              }
350                              else if(inyykh){
351                                     inyykh = false;
352                                    if (yynstates != yykh_count ){
353                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
354                                    }
355                              }
356                              else if(inyyml){
357                                     inyyml = false;
358                                    if (yynstates != yyml_count ){
359                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
360                                    }
361                              }
362                              else if(inyymh){
363                                     inyymh = false;
364                                    if (yynstates != yymh_count ){
365                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
366                                    }
367                              }
368                              else if(inyytl){
369                                     inyytl = false;
370                                    if (yynstates != yytl_count ){
371                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
372                                    }
373                              }
374                              else if(inyyth){
375                                     inyyth = false;
376                                    if (yynstates != yyth_count ){
377                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
378                                    }
379                              }
380                              else if(inyytint){
381                                     inyytint = false;
382                                    if (yyntrans + 1 != yytint_count ){
383                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
384                                    }
385                              }
386                              else if(inyyt){
387                                     inyyt = false;
388                                    if (yyntrans+1 != yyt_count ){
389                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
390                                    }
391                              }
392                              else if(inkeyword){
393                                     inkeyword = false;
394                              }
395                          }else if(line.trim().startsWith("yyk =")){
396                             inyyk = true; 
397                          }else if(line.trim().startsWith("yym =")){
398                             inyym = true;
399                          }else if(line.trim().startsWith("yykl =")){
400                             inyykl = true;
401                          }else if(line.trim().startsWith("yykh =")){
402                             inyykh = true;
403                          }else if(line.trim().startsWith("yyml =")){
404                             inyyml = true;
405                          }else if(line.trim().startsWith("yymh =")){
406                             inyymh = true;
407                          }else if(line.trim().startsWith("yytl =")){
408                             inyytl = true;
409                          }else if(line.trim().startsWith("yyth =")){
410                             inyyth = true;
411                          }else if(line.trim().startsWith("yytint =")){
412                             inyytint = true;
413                          }else if(line.trim().startsWith("yyt =")){
414                             inyyt = true;
415                        }else if(line.trim().startsWith("keywordsvalue =")){
416                           inkeyword = true;
417                        }else if(inyyk){
418                             String[] ss = line.split("[,]");
419                               for(int j=0;j<ss.length;j++){
420                                   // System.out.println(ss[j].trim());
421                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
422                               }
423                          }else if(inyym){
424                               String[] ss = line.split("[,]");
425                                 for(int j=0;j<ss.length;j++){
426                                     // System.out.println(ss[j].trim());
427                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
428                                 }
429                          }else if(inyykl){
430                               String[] ss = line.split("[,]");
431                                 for(int j=0;j<ss.length;j++){
432                                    //  System.out.println(ss[j].trim());
433                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
434                                 }
435                          }else if(inyykh){
436                               String[] ss = line.split("[,]");
437                                 for(int j=0;j<ss.length;j++){
438                                     // System.out.println(ss[j].trim());
439                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
440                                 }
441                          }else if(inyyml){
442                               String[] ss = line.split("[,]");
443                                 for(int j=0;j<ss.length;j++){
444                                     // System.out.println(ss[j].trim());
445                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
446                                 }
447                          }else if(inyymh){
448                               String[] ss = line.split("[,]");
449                                 for(int j=0;j<ss.length;j++){
450                                     // System.out.println(ss[j].trim());
451                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
452                                 }
453                          }else if(inyytl){
454                               String[] ss = line.split("[,]");
455                                 for(int j=0;j<ss.length;j++){
456                                     // System.out.println(ss[j].trim());
457                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
458                                 }
459                          }else if(inyyth){
460                               String[] ss = line.split("[,]");
461                                 for(int j=0;j<ss.length;j++){
462                                     // System.out.println(ss[j].trim());
463                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
464                                 }
465                          }else if(inyytint){
466                               String[] ss = line.split("[,]");
467                                 for(int j=0;j<ss.length;j++){
468                                     // System.out.println(ss[j].trim());
469                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
470                                 }
471                          }else if(inyyt){
472                                //System.out.println(line.trim());
473
474                              c = 0;
475                              String[] st = line.trim().split(",,");
476                              char[] tmp = new char[st.length];
477                              for(int i=0;i<st.length;i++){
478
479                                  if(st[i].startsWith("\'")) {
480                                      if(st[i].length() == 3){  // 'a'
481                                          tmp[c++] = st[i].charAt(1);
482                                      }else if(st[i].length() == 4) { // '\\'
483                                          tmp[c++] = st[i].charAt(2);
484                                      }else{
485                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
486                                      }
487                                  }else{
488                                      try{
489                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
490                                          } catch (NumberFormatException nfe) {
491                                             System.out.println("NumberFormatException: " + nfe.getMessage());
492                                          }
493                                  }
494                              } //while hasmoreTokens
495
496                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
497                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
498                              yyt_count++;
499
500                          }else if(inkeyword){
501                              String[] ss =line.split("[=]");
502
503                              int val1 = -1;
504                              int val2 = -1;
505                              try {
506                                  val1 = Integer.parseInt(ss[1]);
507                                  val2 = Integer.parseInt(ss[2]);
508                              }
509                              catch (NumberFormatException nfe) {
510                                  System.out.println("NumberFormatException: " + nfe.getMessage());
511                              }
512                              keywordValueList.put(ss[0].toUpperCase(),val1);
513                              keywordTypeList.put(val1,val2);
514                             }
515                }
516                }catch(IOException e){
517                  System.out.println(e.toString());
518                }
519                                
520                                yystateLookupConfigure();
521
522    }
523
524
525
526    void yyaction(int yyruleno){
527
528
529
530      int ic;
531      char[] tmparray = {'=','+','-','*','/','>','<'};
532
533      yylvalstr = getyytext();
534  /* actions: */
535  switch(yyruleno){
536  case 1:
537                
538        {
539              addlit(yylvalstr,yytextlen);
540              if (xcdepth <= 0)
541                 {
542                  start(init);
543                  yylvalstr = litbufdup();
544                  returni(cmtslashstar);
545                 }
546              else
547                 xcdepth--;
548
549            break;
550        }
551
552
553  case 2:
554                
555          {
556             // xcdepth++;
557              yyless(1);
558              addlit(yylvalstr,1);
559              break;
560           }
561
562  case 3:
563                        
564          {
565
566              if (getyysstate() == xq)
567              {
568                  nchars = yytextlen;
569                  addlit(yylvalstr, yytextlen-1);
570                  yyless(nchars-1);
571                  return;//exit;
572              }
573
574              xcdepth = 0;
575              start(xc);
576              startlit();
577              yyless(2);
578              addlit(yylvalstr,yytextlen);
579
580          break;
581          }
582
583  case 4:
584                
585          {
586            addlit(yylvalstr,yytextlen);
587            break;
588          }
589
590  case 5:
591                
592          {
593              addlitchar(yylvalstr.charAt(0));
594              break;
595          }
596
597                  
598  case 6:
599                       
600        {
601                                        start(init);
602                                        addlit(yylvalstr, yytextlen);
603                                        yylvalstr = litbufdup();
604                                        returni(ident);
605                break;
606        }
607
608  case 7:
609                        
610        {
611          if ((getyysstate() == xd)||(getyysstate() == xq))
612          {
613              addlit(yylvalstr, 1);
614              yyless(1);
615              return;//exit;
616          }     
617                                        //token_start := yylvalStr;
618                                        start(xbacktick);
619                                        startlit();
620                                        addlit(yylvalstr, yytextlen);
621                break;
622        }
623
624
625  case 8:
626                       
627        {
628                addlit(yylvalstr, yytextlen);
629                break;
630  }
631
632  case 9:
633                       
634        {
635    addlit(yylvalstr, yytextlen);
636                break;
637   }
638                  
639
640  case 10:
641                  
642          {
643            returni(ident);
644            break;
645          }
646          
647  case 11:
648                           
649                        {
650                                if ((getyysstate() == xq)){
651                      addlit(yylvalstr, 1);
652                      yyless(1);
653                      return;
654                                }else{
655                                        returni(sconst);
656                                        break;
657                                }
658                        }
659                        
660  case 12:
661                           
662                        {
663                                if ((getyysstate() == xq)){
664                      addlit(yylvalstr, 1);
665                      yyless(1);
666                      return;
667                                }else{
668                                        returni(TBaseType.teradata_unicode_sconst);
669                                        break;
670                                }
671                        }
672                        
673                        
674  case 13:
675                
676                        {
677                                if ((getyysstate() == xq)){
678                      addlit(yylvalstr, 1);
679                      yyless(1);
680                      return;
681                                }else{
682                                        returni(sconst);
683                                        break;
684                                }
685                        }
686                        
687  case 14:
688                                        
689                        {
690                        returni(sconst);
691                        break;
692                        }                       
693                        
694  case 15:
695                
696                        {
697                        returni(not_equal);
698                        break;
699                        }                       
700
701  case 16:
702                
703          {
704              start(init);
705              addlit(yylvalstr, yytextlen);
706              yylvalstr = litbufdup();
707              returni(sconst);
708              break;
709          }
710
711  case 17:
712                           
713          {
714              start(init);
715              addlit(yylvalstr, yytextlen);
716              yylvalstr = litbufdup();
717              returni(sconst);
718              break;
719          }
720
721  case 18:
722                
723          {
724            addlit(yylvalstr, yytextlen);
725            break;
726          }
727
728
729  case 19:
730                        
731          {
732              if (yysstate == xd)
733                {
734                  addlit(yylvalstr, yytextlen);
735                }
736              else
737                {
738                  start(xq);
739                  startlit();
740                  addlit(yylvalstr, yytextlen);
741                  dummych1 = get_char();
742                  if (dummych1 == '\\') // recognize string like '\'
743                    {
744                      dummych2 = get_char();
745                      if (dummych2 == '\'')
746                      {
747                          start(init);
748                          addlit("\\", 1);
749                          addlit("\'", 1);
750                          yylvalstr = litbufdup();
751                          returni(sconst);
752                      }
753                      else
754                       {
755                          unget_char(dummych2);
756                          unget_char(dummych1);
757                        }
758                    }
759                  else
760                    unget_char(dummych1);
761                }
762
763              break;
764          }
765
766  case 20:
767                
768          {
769              addlit(yylvalstr, yytextlen);
770              break;
771          }
772
773  case 21:
774                
775          {
776            dummych1 = get_char();
777            unget_char(dummych1);
778            if (dummych1 == (char)10)
779              {
780                dummych1 = get_char();
781                addlit(yylvalstr+dummych1, yytextlen+1);
782              } else if (dummych1 == '\\')
783                  {
784                      dummych1 = get_char();
785                      dummych2 = get_char();
786                      if (dummych2 == '\'')
787                        {
788                          unget_char(dummych2);
789                          addlit(yylvalstr+dummych1, yytextlen+1);
790                        }
791                      else
792                        addlit(yylvalstr+dummych1+dummych2, yytextlen+2);
793                  }
794                else
795                  addlit(yylvalstr, yytextlen);
796
797          break;
798         }
799        
800  case 22:
801                 
802          {
803              addlit(yylvalstr, yytextlen);
804              break;
805          }
806
807
808
809  case 23:
810                
811          {
812              start(init);
813              addlit(yylvalstr, yytextlen);
814              if (literallen == 0)   returni (error);
815              if (literallen >= namedatalen)
816              {
817                 setlengthofliteralbuf(namedatalen);
818                 literallen = namedatalen;
819              }
820              yylvalstr = litbufdup();
821              returni (ident);
822
823              break;
824          }
825
826  case 24:
827               
828          {
829          addlit(yylvalstr, yytextlen);
830          break;
831          }
832
833  case 25:
834                        
835          {
836              start(xd);
837              startlit();
838              addlit(yylvalstr, yytextlen);
839              break;
840          }
841        
842  case 26:
843                
844        {
845                
846                dummych1 = get_char();
847                unget_char(dummych1);
848                if (dummych1 == (char)10)
849                  {
850                dummych1 = get_char();
851                addlit(yylvalstr+dummych1, yytextlen+1);
852                  } else
853                    addlit(yylvalstr, yytextlen);
854                
855                break;
856        }
857
858
859  case 27:
860                
861          {
862            returni(lexnewline);
863            break;
864          }
865
866  case 28:
867        
868          {
869              returni(lexspace);
870              break;
871          }
872
873  case 29:
874                
875          {
876          if ((getyysstate() == xq)
877              || (getyysstate() == xd)
878              || (getyysstate() == xc)
879              )
880          {
881              addlit(yylvalstr, 1);
882              yyless(1);
883              return;//exit;
884          }
885
886          returni(cmtdoublehyphen);
887          break;
888          }
889
890  case 30:
891        
892          {
893               returnc(yylvalstr.charAt(0));
894               break;
895          }
896
897  case 31:
898                       
899          {
900            returni(cmpop);
901            break;
902          }
903        
904  case 32:
905                  
906      {
907        returni(concatenationop);
908        break;
909      }
910          
911  case 33:
912           
913          {
914
915          if (getyysstate() == xc)
916             {
917              slashstar = yylvalstr.indexOf("*/");
918                if (slashstar >= 0)
919                  {
920                      addlit(yylvalstr,slashstar+2);
921                      yylvalstr = litbufdup();
922                      yyless(slashstar+2);
923
924                                                                                        if (xcdepth <= 0){
925                                                                                                start(init);
926                                                                                                returni(cmtslashstar);
927                                                                                        }else{
928                                                                                                xcdepth--;
929                                                                                        }
930                  }
931                else
932                  {
933                      addlit(yylvalstr,1);
934                      yyless(1);
935                  }
936             }
937          else
938            {
939              nchars = yytextlen;
940              slashstar = yylvalstr.indexOf("/*");
941              dashdash = yylvalstr.indexOf("--");
942              if ((slashstar >= 0) && (dashdash >= 0))
943                {
944                  //if both appear, take the first one
945                   if (slashstar > dashdash)
946                    {slashstar = dashdash;}
947                }
948              else
949                {
950                  //   if slashstar=0 then slashstar := dashdash;
951                  // add (getyysstate <> xc) to avoid something like this */--,here */ should be handled instead of --
952                  if ((slashstar > 0) && (getyysstate() != xc)) {
953                    nchars = slashstar;
954                    }
955                }
956
957              while ((nchars > 1)
958                      && ( (yylvalstr.charAt(nchars-1) == '+' )
959                          || (yylvalstr.charAt(nchars-1) =='-'))
960                      && (getyysstate() != xc))
961                {
962                  for (ic = nchars - 1; ic>=1; ic--)
963                  {
964                    if (isopchar(yylvalstr.charAt(ic-1)))  break;
965                  }
966                  if (ic >= 1) break;
967                  nchars--;
968                }
969
970              if (nchars < yytextlen)
971                {
972                  //Strip the unwanted chars from the token
973                  yyless(nchars);
974                  yylvalstr = yylvalstr.substring(0,nchars);
975                }
976
977                  ///*
978                  // * If what we have left is only one char, and it's
979                  // * one of the characters matching "self", then
980                  // * return it as a character token the same way
981                  // * that the "self" rule would have.
982                  // * make sure @ return as self char, by james wang
983                  // */
984                  if ((nchars == 1) && (isselfchar(yylvalstr.charAt(0)) || (yylvalstr.charAt(0) == '@')))
985                    {
986                      returnc(yylvalstr.charAt(0));
987                    }
988                  else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '.') )
989                    {
990                        yyless(1);
991                        returnc(yylvalstr.charAt(0));
992                    }
993                  else if (
994                              (nchars >= 2)
995                                  &&(
996                                       charinarray(yylvalstr.charAt(0), tmparray)
997                                       && (yylvalstr.charAt(1) == ':')
998                                       )
999                           )
1000                      {
1001                        yyless(nchars-1);
1002                         yylvalstr = yylvalstr.substring(0,nchars-1);
1003                        if (nchars == 2)
1004                          returnc(yylvalstr.charAt(0));
1005                        else
1006                          returni(op);
1007                      }
1008                    else if (
1009                                (nchars >= 2)
1010                                    && (
1011                                          charinarray(yylvalstr.charAt(nchars-1-1),tmparray)
1012                                          && (yylvalstr.charAt(nchars-1) == '.')
1013                                         )
1014                               )
1015                        {
1016                          yyless(nchars-1);
1017                           yylvalstr = yylvalstr.substring(0,nchars-1);
1018                          if (nchars == 2)
1019                            returnc(yylvalstr.charAt(0));
1020                          else
1021                            returni(op);
1022                        }
1023                    else if ( (nchars == 2) && ((yylvalstr.charAt(nchars-1) == '$')||(yylvalstr.charAt(nchars-1) == '?')) )
1024                      {
1025                        yyless(nchars-1);
1026                        yylvalstr = yylvalstr.substring(0,nchars-1);
1027                        returnc(yylvalstr.charAt(0));
1028                      }
1029                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == '@') )
1030                      {
1031                        if (nchars == 2)
1032                          { // such as >@
1033                              yyless(nchars-1);
1034                            yylvalstr = yylvalstr.substring(0,nchars-1);
1035                             returnc(yylvalstr.charAt(0));
1036                          }
1037                        else
1038                          {
1039                            if (yylvalstr.charAt(nchars-1-1) == '@')
1040                              { //such as >@@,>=@@
1041                                  yyless(nchars-2);
1042                                yylvalstr = yylvalstr.substring(0,nchars-2);
1043                                if (nchars == 3)
1044                                    returnc(yylvalstr.charAt(0));
1045                                else
1046                                    returni(cmpop);
1047                              }
1048                            else
1049                              { // >=@
1050                                  yyless(nchars-1);
1051                                yylvalstr = yylvalstr.substring(0,nchars-1);
1052                                  returni(cmpop);
1053                              }
1054                          }
1055                         }
1056                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == ':') )
1057                      {
1058                        if (nchars == 2)
1059                          { // such as >:
1060                            yyless(nchars-1);
1061                            yylvalstr = yylvalstr.substring(0,nchars-1);
1062                            returnc(yylvalstr.charAt(0));
1063                          }
1064                        else
1065                          {
1066                          // >=:
1067                                  yyless(nchars-1);
1068                                yylvalstr = yylvalstr.substring(0,nchars-1);
1069                                                        if (yylvalstr.equalsIgnoreCase("||")){
1070                                  returni(concatenationop);
1071                              }else                                                             
1072                                  returni(cmpop);
1073                         }
1074                       
1075                         }
1076                                         else if ( (nchars >= 2) && (yylvalstr.charAt(0)== '|') && (yylvalstr.charAt(1) == '|'))
1077                        { //||--sss ,just get || only
1078                              yyless(2);
1079                              yylvalstr = yylvalstr.substring(0,2);
1080                              returni(concatenationop); //return as concatenationop, this is used in sybase only
1081                        }
1082                      else if ((nchars == 2) && ((yylvalstr.charAt(0) =='.') && ( (yylvalstr.charAt(1)=='*') || (yylvalstr.charAt(1) =='#')) ))
1083                          {
1084                            yyless(1);
1085                            returnc(yylvalstr.charAt(0));
1086                          }
1087                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '#')) ))
1088                          {
1089                            yyless(1);
1090                            returnc(yylvalstr.charAt(0));
1091                          }
1092                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '?')) ))
1093                          {
1094                            yyless(1);
1095                            returnc(yylvalstr.charAt(0));
1096                          }
1097                                        else if ( (nchars >= 3) && (yylvalstr.charAt(nchars-1) == '$' ) && (yylvalstr.charAt(nchars-2) == '$' ))
1098                                        { // =$$abc
1099                                            if (nchars == 3){
1100                                                        yyless(1);
1101                                                        returnc(yylvalstr.charAt(0));
1102                                                }else{
1103                                                        yyless(nchars-2);
1104                                                        yylvalstr = yylvalstr.substring(0,nchars-2);
1105                                                        returni(cmpop);
1106                                                }
1107                                        }
1108                                                  else if (((nchars > 2) && (yylvalstr.charAt(0) == '*'))
1109                              && (yylvalstr.charAt(1) == '/')
1110                              && (getyysstate() == xc)
1111                              )
1112                           { //in comment, and find */ , then it must the end of comment
1113                             yyless(2);
1114                                addlit(yylvalstr,yytextlen);
1115                             if (xcdepth <= 0)
1116                               {
1117                                 start(init);
1118                                 yylvalstr = litbufdup();
1119                                 returni(cmtslashstar);
1120                               }
1121                             else
1122                               xcdepth--;
1123                           }
1124                      else
1125                        returni(op);
1126               }
1127
1128          break;
1129          }
1130
1131  case 34:
1132                        
1133          {
1134               returni(iconst);
1135               break;
1136          }
1137
1138  case 35:
1139                        
1140          {
1141            ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
1142            nchars = yytextlen;
1143            if (yylvalstr.charAt(nchars-1) == '.')
1144              {
1145                dummych1 = get_char();
1146                unget_char(dummych1);
1147                if (dummych1 == '.')
1148                  {
1149                      yyless(nchars-1);
1150                    yylvalstr = yylvalstr.substring(0,nchars - 1);
1151                    returni (iconst);
1152                    return;//exit;
1153                  }
1154              }
1155            returni (fconst);
1156           break;
1157          }
1158
1159  case 36:
1160                        
1161          {
1162           returni (fconst);
1163           break;
1164          }
1165
1166  case 37:
1167              
1168          {
1169            int rw;
1170            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1171            else
1172               returni(ident);
1173            break;
1174          }
1175
1176
1177  case 38:
1178           
1179        {
1180                if (yylvalstr.charAt(0) == '$'){
1181                        returni(ident);
1182                }else
1183                        returni(variable);
1184
1185                break;
1186        }
1187                  
1188  case 39:
1189                  
1190          {
1191            if ((yylvalstr.charAt(0) == '$')&&(yylvalstr.charAt(1) == '_')){
1192              returni(ident);
1193            }else
1194              returni(variable);
1195
1196            break;
1197          }
1198 
1199  case 40:
1200           
1201          {
1202            returni(mslabel);
1203            break;
1204          }
1205
1206                  
1207  case 41:
1208               
1209          {
1210            returni(lexspace);
1211            break;
1212          }               
1213                  
1214  case 42:
1215                    
1216          {
1217            returni(UNICODE_ENCODE_ID);
1218            break;
1219          }               
1220                  
1221    
1222  case 43:
1223                        
1224          {
1225            returni( error);
1226            break;
1227          }
1228    default:{
1229     System.out.println("fatal error in yyaction");
1230    }
1231   }//switch
1232}/*yyaction*/;
1233
1234
1235
1236        }