Source code

001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************}
004{   Lexical analizer for GSQLParser component java version    }
005{   Copyright (c) 2004-2025 by Gudu software              }
006{****************************************************/
007
008package gudusoft.gsqlparser;
009
010
011import java.util.*;
012import java.io.InputStreamReader;
013
014import java.io.BufferedReader;
015import java.io.IOException;
016
017import gudusoft.gsqlparser.stmt.teradata.utilities.*;
018
019public class TLexerTeradata extends TCustomLexer{
020    static int yynmarks = 0  ;
021    static int yynmatches ;
022    static int yyntrans   ;
023    static int yynstates  ;
024    static int[]  yyk,yym ; // 1 based
025    static int[]  yytint;  // 1 based
026    static TYytRec[] yyt ;  // 1 based
027    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
028    private  static  String[] keywordlist;
029    static String   table_file;
030    static HashMap<String, Integer> keywordValueList;
031    static HashMap<Integer, Integer> keywordTypeList;
032          static String  bteqCmdStr,multiLoadCmdStr,fastExportCmdStr,fastLoadCmdStr;
033    static List<String> bteqCmdList,multiLoadCmdList,fastExportCmdList,fastLoadCmdList;
034        static int[][] yystateTable;
035        static HashMap<String, Integer> operatorList;
036
037    static {
038                        table_file = "/gudusoft/gsqlparser/parser/teradata/teradata_lex_table.txt";
039      keywordValueList = new HashMap<String, Integer>();
040              keywordTypeList = new HashMap<Integer, Integer>();
041                        bteqCmdList = new ArrayList<String>( );
042                        multiLoadCmdList = new ArrayList<String>( );
043                        fastExportCmdList = new ArrayList<String>( );
044                        fastLoadCmdList = new ArrayList<String>( );
045
046                        // Initialize operator list for proper tokenization of operators followed by +/-
047                        operatorList = new HashMap<String, Integer>();
048                        operatorList.put("+", Integer.valueOf('+'));
049                        operatorList.put("-", Integer.valueOf('-'));
050                        operatorList.put("*", Integer.valueOf('*'));
051                        operatorList.put("/", Integer.valueOf('/'));
052                        operatorList.put("%", Integer.valueOf('%'));
053                        operatorList.put("^", Integer.valueOf('^'));
054                        operatorList.put("&", Integer.valueOf('&'));
055                        operatorList.put("|", Integer.valueOf('|'));
056                        operatorList.put("~", Integer.valueOf('~'));
057                        operatorList.put("!", Integer.valueOf('!'));
058                        operatorList.put(">", Integer.valueOf('>'));
059                        operatorList.put("<", Integer.valueOf('<'));
060                        operatorList.put("=", Integer.valueOf('='));
061                        operatorList.put("<=", TBaseType.less_equal);
062                        operatorList.put("<>", TBaseType.not_equal);
063                        operatorList.put(">=", TBaseType.great_equal);
064                        operatorList.put("!=", TBaseType.not_equal);
065                        operatorList.put("^=", TBaseType.not_equal);
066                        operatorList.put("^<", TBaseType.not_less);
067                        operatorList.put("^>", TBaseType.not_great);
068                        operatorList.put("!<", TBaseType.not_less);   // SQL Server style not less than
069                        operatorList.put("!>", TBaseType.not_great);  // SQL Server style not greater than
070                        operatorList.put("**", TBaseType.exponentiate);
071                        operatorList.put("||", TBaseType.concatenationop);
072
073                        if (TBaseType.enterprise_edition||TBaseType.teradata_edition){
074                                inittable();
075                        }
076    }
077
078    public TLexerTeradata(){
079          super();
080          dbvendor = EDbVendor.dbvteradata;
081          sourcetokens = new TSourceTokenList();
082    }
083
084    public void setTeradataUtilityType(TeradataUtilityType teradataUtilityType) {
085        this.teradataUtilityType = teradataUtilityType;
086    }
087
088    private TeradataUtilityType teradataUtilityType = TeradataUtilityType.BTEQ;
089    
090    public TSourceTokenList sourcetokens;
091
092
093
094    int issystemvariable(String str){
095        return -1;// -1 means not a system variable
096    }
097
098public boolean canBeColumnName(int tokencode){
099    //http://blog.csdn.net/superbeck/article/details/5387476
100    boolean ret = false;
101    int modifiers = keyword_type_identifier | keyword_type_column ;
102    Integer s = keywordTypeList.get(tokencode);
103     if (s != null){
104        int modifier = s;
105        ret = (modifiers & modifier) == modifier;
106    }
107
108    return ret;
109}
110
111   public EFindSqlStateType cmdType(TSourceToken token){
112        
113        // use  isMultiLoadCmd,isFastExportCmd,isFastLoadCmd,isBteqCmd to find the command type,
114        String str = token.toString();
115        String str2 = null;
116        String str3 = null;
117        if (token.nextSolidToken() != null) {
118            str2 = token.nextSolidToken().toString();
119            if (token.nextSolidToken().nextSolidToken() != null) {
120              str3 = token.nextSolidToken().nextSolidToken().toString();
121            }
122        }
123
124    // First check the longest possible command (str + str2 + str3)
125    if (str3 != null) {
126        String combinedStr3 = str + " " + str2 + " " + str3;
127        int matchCount = 0;
128        boolean isMultiLoad3 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr3) != null;
129        boolean isFastExport3 = FastExportCmdType.searchFastExportCmd(combinedStr3) != null;
130        boolean isFastLoad3 = FastLoadCmdType.searchFastLoadCmd(combinedStr3) != null;
131        boolean isBteq3 = BteqCmdType.searchBteqCmd(combinedStr3) != null;
132
133        if (isMultiLoad3) matchCount++;
134        if (isFastExport3) matchCount++;
135        if (isFastLoad3) matchCount++;
136        if (isBteq3) matchCount++;
137
138        if (matchCount > 1) {
139            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq3) return EFindSqlStateType.stBTEQCmd;
140            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
141            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport3) return EFindSqlStateType.stFastExportCmd;
142            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
143
144            // throw new IllegalStateException("Ambiguous command '" + combinedStr3 + "' matches multiple utility types");
145            matchCount = 1;
146        }
147
148        if (matchCount == 1) {
149            if (isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
150            if (isFastExport3) return EFindSqlStateType.stFastExportCmd;
151            if (isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
152            if (isBteq3) return EFindSqlStateType.stBTEQCmd;
153        }
154    }
155
156    // Then check medium length command (str + str2)
157    if (str2 != null) {
158        String combinedStr2 = str + " " + str2;
159        int matchCount = 0;
160        boolean isMultiLoad2 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr2) != null;
161        boolean isFastExport2 = FastExportCmdType.searchFastExportCmd(combinedStr2) != null;
162        boolean isFastLoad2 = FastLoadCmdType.searchFastLoadCmd(combinedStr2) != null;
163        boolean isBteq2 = BteqCmdType.searchBteqCmd(combinedStr2) != null;
164
165        if (isMultiLoad2) matchCount++;
166        if (isFastExport2) matchCount++;
167        if (isFastLoad2) matchCount++;
168        if (isBteq2) matchCount++;
169
170        if (matchCount > 1) {
171            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq2) return EFindSqlStateType.stBTEQCmd;
172            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
173            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport2) return EFindSqlStateType.stFastExportCmd;
174            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
175
176            //throw new IllegalStateException("Ambiguous command '" + combinedStr2 + "' matches multiple utility types");
177            matchCount = 1;
178        }
179        if (matchCount == 1) {
180            if (isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
181            if (isFastExport2) return EFindSqlStateType.stFastExportCmd;
182            if (isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
183            if (isBteq2) return EFindSqlStateType.stBTEQCmd;
184        }
185    }
186
187    // Finally check single word command (str)
188    boolean isMultiLoad = MultiLoadCmdType.searchMultiLoadCmd(str) != null;
189    boolean isFastExport = FastExportCmdType.searchFastExportCmd(str) != null;
190    boolean isFastLoad = FastLoadCmdType.searchFastLoadCmd(str) != null;
191    boolean isBteq = BteqCmdType.searchBteqCmd(str) != null;
192    int matchCount = 0;
193    if (isMultiLoad) matchCount++;
194    if (isFastExport) matchCount++;
195    if (isFastLoad) matchCount++;
196    if (isBteq) matchCount++;
197
198    if (matchCount > 1) {
199        if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq) return EFindSqlStateType.stBTEQCmd;
200        if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
201        if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport) return EFindSqlStateType.stFastExportCmd;
202        if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad) return EFindSqlStateType.stFastLoadCmd;
203
204       // throw new IllegalStateException("Ambiguous command '" + str + "' matches multiple utility types");
205        matchCount = 1;
206    }
207
208    if (matchCount == 1) {
209      if (isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
210      if (isFastExport) return EFindSqlStateType.stFastExportCmd;
211      if (isFastLoad) return EFindSqlStateType.stFastLoadCmd;
212      if (isBteq) return EFindSqlStateType.stBTEQCmd;
213  }
214    return EFindSqlStateType.stnormal;
215
216  }
217
218
219
220
221 public  int iskeyword(String str){
222        int ret = -1;
223        Integer s = keywordValueList.get(str.toUpperCase(Locale.ENGLISH));
224        if( s != null){
225            ret = s;
226        }
227        return ret;// -1 means not a keyword
228     }
229         
230    public static EKeywordType getKeywordType(String keyword){
231        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
232    }            
233
234     public int getkeywordvalue(String keyword){
235        int ret = 0;
236        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
237        if( s != null){
238            ret = s;
239         }
240        return ret;// 0 means not a keyword
241     }
242
243    static void yystateLookupConfigure() {
244        int yystates = yytl.length;
245        yystateTable = new int[257][yystates];
246
247        // initialize to empty
248        for(int i = 0; i < yystates; i++) {
249            for (int j = 0; j < 257; j++)
250                yystateTable[j][i] = -1;
251        }
252
253        for(int i = 0; i < yystates; i++) {
254            int low = yytl[i];
255            int high = yyth[i];
256            for (int j = low; j <= high; j++) {
257                for (char c: yyt[j].cc) {
258                    yystateTable[c][i] = j;
259                }
260            }
261        }
262    }
263        
264    int yylex(){
265          int yyn;
266           while (true) { // top level while
267              yynew();
268              while (true){  //scan
269                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
270                     yymark(yyk[yyn]);
271                 }
272
273                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
274                    yymatch(yym[yyn]);
275                 }
276
277                 if(yytl[yystate] > yyth[yystate]){
278                     break;
279                 }
280
281                 yyscan();
282//                 yyn = yytl[yystate];
283                 totablechar();
284//                 while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
285//                   yyn++;
286//                 }
287//                 if (yyn > yyth[yystate]){
288//                     break;
289//                 }
290
291                 yyn = yystateTable[yytablechar][yystate];
292                 if (yyn == -1)
293                     break;
294                                         
295                 yystate = yyt[yyn].s;
296              } //scan
297
298              while (true){ //action
299                int yyrule;
300                if ( (yyrule = yyfind()) != -1 ){
301                   yyaction(yyrule);
302                   if (yyreject){
303                       continue;
304                   }
305                }else if( (!yydefault() ) && (yywrap()) ){
306                   yyclear();
307                   returni(0);
308                }
309                break;
310              }
311
312              if (!yydone) {
313                  continue;
314              }
315              break;
316            } // top level while
317
318           return yyretval;
319        }
320
321    
322
323    static void inittable(){
324                
325                //if (yynmarks > 0) return; //init table already
326
327        String line;
328        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
329        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
330        int c=0;
331        keywordValueList.clear();
332        keywordTypeList.clear();
333        
334        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerTeradata.class.getResourceAsStream(table_file)));
335
336            try{
337                while( (line = br.readLine()) != null){
338                          if (line.trim().startsWith("yynmarks=")){
339                             String[] ss = line.split("[=;]");
340                              yynmarks=Integer.parseInt(ss[1].trim());
341                              yyk = new int[yynmarks+1];
342                          }else if (line.trim().startsWith("yynmatches=")){
343                              String[] ss = line.split("[=;]");
344                               yynmatches=Integer.parseInt(ss[1].trim());
345                               yym = new int[yynmatches+1];
346                          }else if (line.trim().startsWith("yyntrans=")){
347                              String[] ss = line.split("[=;]");
348                               yyntrans=Integer.parseInt(ss[1].trim());
349                               yytint = new int[yyntrans+1];
350                               yyt = new TYytRec[yyntrans+1];
351                          }else if (line.trim().startsWith("yynstates=")){
352                              String[] ss = line.split("[=;]");
353                               yynstates=Integer.parseInt(ss[1].trim());
354                               yykl = new int[yynstates];
355                               yykh = new int[yynstates];
356                              yyml = new int[yynstates];
357                              yymh = new int[yynstates];
358                              yytl = new int[yynstates];
359                              yyth = new int[yynstates];
360                          }else if (line.trim().startsWith("<end>")){
361                              if (inyyk){
362                                  inyyk = false;
363                                 if (yynmarks+1 != yyk_count ){
364                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
365                                 }
366                              }
367                              else if(inyym){
368                                     inyym = false;
369                                    if (yynmatches+1 != yym_count ){
370                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
371                                    }
372                              }
373                              else if(inyykl){
374                                     inyykl = false;
375                                    if (yynstates != yykl_count ){
376                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
377                                    }
378                              }
379                              else if(inyykh){
380                                     inyykh = false;
381                                    if (yynstates != yykh_count ){
382                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
383                                    }
384                              }
385                              else if(inyyml){
386                                     inyyml = false;
387                                    if (yynstates != yyml_count ){
388                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
389                                    }
390                              }
391                              else if(inyymh){
392                                     inyymh = false;
393                                    if (yynstates != yymh_count ){
394                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
395                                    }
396                              }
397                              else if(inyytl){
398                                     inyytl = false;
399                                    if (yynstates != yytl_count ){
400                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
401                                    }
402                              }
403                              else if(inyyth){
404                                     inyyth = false;
405                                    if (yynstates != yyth_count ){
406                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
407                                    }
408                              }
409                              else if(inyytint){
410                                     inyytint = false;
411                                    if (yyntrans + 1 != yytint_count ){
412                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
413                                    }
414                              }
415                              else if(inyyt){
416                                     inyyt = false;
417                                    if (yyntrans+1 != yyt_count ){
418                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
419                                    }
420                              }
421                              else if(inkeyword){
422                                     inkeyword = false;
423                              }
424                          }else if(line.trim().startsWith("yyk =")){
425                             inyyk = true; 
426                          }else if(line.trim().startsWith("yym =")){
427                             inyym = true;
428                          }else if(line.trim().startsWith("yykl =")){
429                             inyykl = true;
430                          }else if(line.trim().startsWith("yykh =")){
431                             inyykh = true;
432                          }else if(line.trim().startsWith("yyml =")){
433                             inyyml = true;
434                          }else if(line.trim().startsWith("yymh =")){
435                             inyymh = true;
436                          }else if(line.trim().startsWith("yytl =")){
437                             inyytl = true;
438                          }else if(line.trim().startsWith("yyth =")){
439                             inyyth = true;
440                          }else if(line.trim().startsWith("yytint =")){
441                             inyytint = true;
442                          }else if(line.trim().startsWith("yyt =")){
443                             inyyt = true;
444                        }else if(line.trim().startsWith("keywordsvalue =")){
445                           inkeyword = true;
446                        }else if(inyyk){
447                             String[] ss = line.split("[,]");
448                               for(int j=0;j<ss.length;j++){
449                                   // System.out.println(ss[j].trim());
450                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
451                               }
452                          }else if(inyym){
453                               String[] ss = line.split("[,]");
454                                 for(int j=0;j<ss.length;j++){
455                                     // System.out.println(ss[j].trim());
456                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
457                                 }
458                          }else if(inyykl){
459                               String[] ss = line.split("[,]");
460                                 for(int j=0;j<ss.length;j++){
461                                    //  System.out.println(ss[j].trim());
462                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
463                                 }
464                          }else if(inyykh){
465                               String[] ss = line.split("[,]");
466                                 for(int j=0;j<ss.length;j++){
467                                     // System.out.println(ss[j].trim());
468                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
469                                 }
470                          }else if(inyyml){
471                               String[] ss = line.split("[,]");
472                                 for(int j=0;j<ss.length;j++){
473                                     // System.out.println(ss[j].trim());
474                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
475                                 }
476                          }else if(inyymh){
477                               String[] ss = line.split("[,]");
478                                 for(int j=0;j<ss.length;j++){
479                                     // System.out.println(ss[j].trim());
480                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
481                                 }
482                          }else if(inyytl){
483                               String[] ss = line.split("[,]");
484                                 for(int j=0;j<ss.length;j++){
485                                     // System.out.println(ss[j].trim());
486                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
487                                 }
488                          }else if(inyyth){
489                               String[] ss = line.split("[,]");
490                                 for(int j=0;j<ss.length;j++){
491                                     // System.out.println(ss[j].trim());
492                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
493                                 }
494                          }else if(inyytint){
495                               String[] ss = line.split("[,]");
496                                 for(int j=0;j<ss.length;j++){
497                                     // System.out.println(ss[j].trim());
498                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
499                                 }
500                          }else if(inyyt){
501                                //System.out.println(line.trim());
502
503                              c = 0;
504                              String[] st = line.trim().split(",,");
505                              char[] tmp = new char[st.length];
506                              for(int i=0;i<st.length;i++){
507
508                                  if(st[i].startsWith("\'")) {
509                                      if(st[i].length() == 3){  // 'a'
510                                          tmp[c++] = st[i].charAt(1);
511                                      }else if(st[i].length() == 4) { // '\\'
512                                          tmp[c++] = st[i].charAt(2);
513                                      }else{
514                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
515                                      }
516                                  }else{
517                                      try{
518                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
519                                          } catch (NumberFormatException nfe) {
520                                             System.out.println("NumberFormatException: " + nfe.getMessage());
521                                          }
522                                  }
523                              } //while hasmoreTokens
524
525                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
526                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
527                              yyt_count++;
528
529                          }else if(inkeyword){
530                              String[] ss =line.split("[=]");
531
532                              int val1 = -1;
533                              int val2 = -1;
534                              try {
535                                  val1 = Integer.parseInt(ss[1]);
536                                  val2 = Integer.parseInt(ss[2]);
537                              }
538                              catch (NumberFormatException nfe) {
539                                  System.out.println("NumberFormatException: " + nfe.getMessage());
540                              }
541                              keywordValueList.put(ss[0].toUpperCase(),val1);
542                              keywordTypeList.put(val1,val2);
543                             }
544                }
545                }catch(IOException e){
546                  System.out.println(e.toString());
547                }
548                                
549                                yystateLookupConfigure();
550
551    }
552
553
554
555    void yyaction(int yyruleno){
556
557
558
559      int ic;
560      char[] tmparray = {'=','+','-','*','/','>','<'};
561
562      yylvalstr = getyytext();
563  /* actions: */
564  switch(yyruleno){
565  case 1:
566                
567        {
568              addlit(yylvalstr,yytextlen);
569              if (xcdepth <= 0)
570                 {
571                  start(init);
572                  yylvalstr = litbufdup();
573                  returni(cmtslashstar);
574                 }
575              else
576                 xcdepth--;
577
578            break;
579        }
580
581
582  case 2:
583                
584          {
585             // xcdepth++;
586              yyless(1);
587              addlit(yylvalstr,1);
588              break;
589           }
590
591  case 3:
592                        
593          {
594
595              if (getyysstate() == xq)
596              {
597                  nchars = yytextlen;
598                  addlit(yylvalstr, yytextlen-1);
599                  yyless(nchars-1);
600                  return;//exit;
601              }
602
603              xcdepth = 0;
604              start(xc);
605              startlit();
606              yyless(2);
607              addlit(yylvalstr,yytextlen);
608
609          break;
610          }
611
612  case 4:
613                
614          {
615            addlit(yylvalstr,yytextlen);
616            break;
617          }
618
619  case 5:
620                
621          {
622              addlitchar(yylvalstr.charAt(0));
623              break;
624          }
625
626                  
627  case 6:
628                       
629        {
630                                        start(init);
631                                        addlit(yylvalstr, yytextlen);
632                                        yylvalstr = litbufdup();
633                                        returni(ident);
634                break;
635        }
636
637  case 7:
638                        
639        {
640          if ((getyysstate() == xd)||(getyysstate() == xq))
641          {
642              addlit(yylvalstr, 1);
643              yyless(1);
644              return;//exit;
645          }     
646                                        //token_start := yylvalStr;
647                                        start(xbacktick);
648                                        startlit();
649                                        addlit(yylvalstr, yytextlen);
650                break;
651        }
652
653
654  case 8:
655                       
656        {
657                addlit(yylvalstr, yytextlen);
658                break;
659  }
660
661  case 9:
662                       
663        {
664    addlit(yylvalstr, yytextlen);
665                break;
666   }
667                  
668
669  case 10:
670                  
671          {
672            returni(ident);
673            break;
674          }
675          
676  case 11:
677                           
678                        {
679                                if ((getyysstate() == xq)){
680                      addlit(yylvalstr, 1);
681                      yyless(1);
682                      return;
683                                }else{
684                                        returni(sconst);
685                                        break;
686                                }
687                        }
688                        
689  case 12:
690                           
691                        {
692                                if ((getyysstate() == xq)){
693                      addlit(yylvalstr, 1);
694                      yyless(1);
695                      return;
696                                }else{
697                                        returni(TBaseType.teradata_unicode_sconst);
698                                        break;
699                                }
700                        }
701                        
702                        
703  case 13:
704                
705                        {
706                                if ((getyysstate() == xq)){
707                      addlit(yylvalstr, 1);
708                      yyless(1);
709                      return;
710                                }else{
711                                        returni(sconst);
712                                        break;
713                                }
714                        }
715                        
716  case 14:
717                                        
718                        {
719                        returni(sconst);
720                        break;
721                        }                       
722                        
723  case 15:
724                
725                        {
726                        returni(not_equal);
727                        break;
728                        }                       
729
730  case 16:
731                
732          {
733              start(init);
734              addlit(yylvalstr, yytextlen);
735              yylvalstr = litbufdup();
736              returni(sconst);
737              break;
738          }
739
740  case 17:
741                           
742          {
743              start(init);
744              addlit(yylvalstr, yytextlen);
745              yylvalstr = litbufdup();
746              returni(sconst);
747              break;
748          }
749
750  case 18:
751                
752          {
753            addlit(yylvalstr, yytextlen);
754            break;
755          }
756
757
758  case 19:
759                        
760          {
761              if (yysstate == xd)
762                {
763                  addlit(yylvalstr, yytextlen);
764                }
765              else
766                {
767                  start(xq);
768                  startlit();
769                  addlit(yylvalstr, yytextlen);
770                  dummych1 = get_char();
771                  if (dummych1 == '\\') // recognize string like '\'
772                    {
773                      dummych2 = get_char();
774                      if (dummych2 == '\'')
775                      {
776                          start(init);
777                          addlit("\\", 1);
778                          addlit("\'", 1);
779                          yylvalstr = litbufdup();
780                          returni(sconst);
781                      }
782                      else
783                       {
784                          unget_char(dummych2);
785                          unget_char(dummych1);
786                        }
787                    }
788                  else
789                    unget_char(dummych1);
790                }
791
792              break;
793          }
794
795  case 20:
796                
797          {
798              addlit(yylvalstr, yytextlen);
799              break;
800          }
801
802  case 21:
803                
804          {
805            dummych1 = get_char();
806            unget_char(dummych1);
807            if (dummych1 == (char)10)
808              {
809                dummych1 = get_char();
810                addlit(yylvalstr+dummych1, yytextlen+1);
811              } else if (dummych1 == '\\')
812                  {
813                      dummych1 = get_char();
814                      dummych2 = get_char();
815                      if (dummych2 == '\'')
816                        {
817                          unget_char(dummych2);
818                          addlit(yylvalstr+dummych1, yytextlen+1);
819                        }
820                      else
821                        addlit(yylvalstr+dummych1+dummych2, yytextlen+2);
822                  }
823                else
824                  addlit(yylvalstr, yytextlen);
825
826          break;
827         }
828        
829  case 22:
830                 
831          {
832              addlit(yylvalstr, yytextlen);
833              break;
834          }
835
836
837
838  case 23:
839                
840          {
841              start(init);
842              addlit(yylvalstr, yytextlen);
843              if (literallen == 0)   returni (error);
844              if (literallen >= namedatalen)
845              {
846                 setlengthofliteralbuf(namedatalen);
847                 literallen = namedatalen;
848              }
849              yylvalstr = litbufdup();
850              returni (ident);
851
852              break;
853          }
854
855  case 24:
856               
857          {
858          addlit(yylvalstr, yytextlen);
859          break;
860          }
861
862  case 25:
863           
864          {
865              /* Unicode escape identifier U&"..." - treated same as double-quoted identifier */
866              start(xd);
867              startlit();
868              addlit(yylvalstr, yytextlen);
869              break;
870          }
871
872  case 26:
873         
874          {
875              start(xd);
876              startlit();
877              addlit(yylvalstr, yytextlen);
878              break;
879          }
880        
881  case 27:
882                
883        {
884                
885                dummych1 = get_char();
886                unget_char(dummych1);
887                if (dummych1 == (char)10)
888                  {
889                dummych1 = get_char();
890                addlit(yylvalstr+dummych1, yytextlen+1);
891                  } else
892                    addlit(yylvalstr, yytextlen);
893                
894                break;
895        }
896
897
898  case 28:
899                
900          {
901            returni(lexnewline);
902            break;
903          }
904
905  case 29:
906        
907          {
908              returni(lexspace);
909              break;
910          }
911
912  case 30:
913                
914          {
915          if ((getyysstate() == xq)
916              || (getyysstate() == xd)
917              || (getyysstate() == xc)
918              )
919          {
920              addlit(yylvalstr, 1);
921              yyless(1);
922              return;//exit;
923          }
924
925          returni(cmtdoublehyphen);
926          break;
927          }
928
929  case 31:
930                 
931          {
932               returni(ident);
933               break;
934          }
935
936  case 32:
937      
938          {
939               returnc(yylvalstr.charAt(0));
940               break;
941          }
942
943  case 33:
944                       
945          {
946            returni(cmpop);
947            break;
948          }
949        
950  case 34:
951                  
952      {
953        returni(concatenationop);
954        break;
955      }
956          
957  case 35:
958           
959          {
960
961          if (getyysstate() == xc)
962             {
963              slashstar = yylvalstr.indexOf("*/");
964                if (slashstar >= 0)
965                  {
966                      addlit(yylvalstr,slashstar+2);
967                      yylvalstr = litbufdup();
968                      yyless(slashstar+2);
969
970                                                                                        if (xcdepth <= 0){
971                                                                                                start(init);
972                                                                                                returni(cmtslashstar);
973                                                                                        }else{
974                                                                                                xcdepth--;
975                                                                                        }
976                  }
977                else
978                  {
979                      addlit(yylvalstr,1);
980                      yyless(1);
981                  }
982             }
983          else
984            {
985              nchars = yytextlen;
986              slashstar = yylvalstr.indexOf("/*");
987              dashdash = yylvalstr.indexOf("--");
988              if ((slashstar >= 0) && (dashdash >= 0))
989                {
990                  //if both appear, take the first one
991                   if (slashstar > dashdash)
992                    {slashstar = dashdash;}
993                }
994              else
995                {
996                  //   if slashstar=0 then slashstar := dashdash;
997                  // add (getyysstate <> xc) to avoid something like this */--,here */ should be handled instead of --
998                  if ((slashstar > 0) && (getyysstate() != xc)) {
999                    nchars = slashstar;
1000                    }
1001                }
1002
1003              // Search for known operators from longest to shortest
1004              // This handles cases like <=-.01 where <= should be returned, not <=-
1005              boolean searchOp = false;
1006              while (nchars > 0){
1007                  String opStr = yylvalstr.substring(0,nchars);
1008                  Integer opCode = operatorList.get(opStr);
1009                  if (opCode != null){
1010                      if (nchars < yytextlen){
1011                          yyless(nchars);
1012                          yylvalstr = yylvalstr.substring(0,nchars);
1013                      }
1014                      searchOp = true;
1015                      returni(opCode);
1016                      break;
1017                  }
1018                  nchars--;
1019              }
1020
1021              if (searchOp) break;
1022
1023              nchars = yytextlen; // reset nchars
1024
1025              while ((nchars > 1)
1026                      && ( (yylvalstr.charAt(nchars-1) == '+' )
1027                          || (yylvalstr.charAt(nchars-1) =='-'))
1028                      && (getyysstate() != xc))
1029                {
1030                  for (ic = nchars - 1; ic>=1; ic--)
1031                  {
1032                    if (isopchar(yylvalstr.charAt(ic-1)))  break;
1033                  }
1034                  if (ic >= 1) break;
1035                  nchars--;
1036                }
1037
1038              if (nchars < yytextlen)
1039                {
1040                  //Strip the unwanted chars from the token
1041                  yyless(nchars);
1042                  yylvalstr = yylvalstr.substring(0,nchars);
1043                }
1044
1045                  ///*
1046                  // * If what we have left is only one char, and it's
1047                  // * one of the characters matching "self", then
1048                  // * return it as a character token the same way
1049                  // * that the "self" rule would have.
1050                  // * make sure @ return as self char, by james wang
1051                  // */
1052                  if ((nchars == 1) && (isselfchar(yylvalstr.charAt(0)) || (yylvalstr.charAt(0) == '@')))
1053                    {
1054                      returnc(yylvalstr.charAt(0));
1055                    }
1056                  else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '.') )
1057                    {
1058                        yyless(1);
1059                        returnc(yylvalstr.charAt(0));
1060                    }
1061                  else if (
1062                              (nchars >= 2)
1063                                  &&(
1064                                       charinarray(yylvalstr.charAt(0), tmparray)
1065                                       && (yylvalstr.charAt(1) == ':')
1066                                       )
1067                           )
1068                      {
1069                        yyless(nchars-1);
1070                         yylvalstr = yylvalstr.substring(0,nchars-1);
1071                        if (nchars == 2)
1072                          returnc(yylvalstr.charAt(0));
1073                        else
1074                          returni(op);
1075                      }
1076                    else if (
1077                                (nchars >= 2)
1078                                    && (
1079                                          charinarray(yylvalstr.charAt(nchars-1-1),tmparray)
1080                                          && (yylvalstr.charAt(nchars-1) == '.')
1081                                         )
1082                               )
1083                        {
1084                          yyless(nchars-1);
1085                           yylvalstr = yylvalstr.substring(0,nchars-1);
1086                          if (nchars == 2)
1087                            returnc(yylvalstr.charAt(0));
1088                          else
1089                            returni(op);
1090                        }
1091                    else if ( (nchars == 2) && ((yylvalstr.charAt(nchars-1) == '$')||(yylvalstr.charAt(nchars-1) == '?')) )
1092                      {
1093                        yyless(nchars-1);
1094                        yylvalstr = yylvalstr.substring(0,nchars-1);
1095                        returnc(yylvalstr.charAt(0));
1096                      }
1097                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == '@') )
1098                      {
1099                        if (nchars == 2)
1100                          { // such as >@
1101                              yyless(nchars-1);
1102                            yylvalstr = yylvalstr.substring(0,nchars-1);
1103                             returnc(yylvalstr.charAt(0));
1104                          }
1105                        else
1106                          {
1107                            if (yylvalstr.charAt(nchars-1-1) == '@')
1108                              { //such as >@@,>=@@
1109                                  yyless(nchars-2);
1110                                yylvalstr = yylvalstr.substring(0,nchars-2);
1111                                if (nchars == 3)
1112                                    returnc(yylvalstr.charAt(0));
1113                                else
1114                                    returni(cmpop);
1115                              }
1116                            else
1117                              { // >=@
1118                                  yyless(nchars-1);
1119                                yylvalstr = yylvalstr.substring(0,nchars-1);
1120                                  returni(cmpop);
1121                              }
1122                          }
1123                         }
1124                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == ':') )
1125                      {
1126                        if (nchars == 2)
1127                          { // such as >:
1128                            yyless(nchars-1);
1129                            yylvalstr = yylvalstr.substring(0,nchars-1);
1130                            returnc(yylvalstr.charAt(0));
1131                          }
1132                        else
1133                          {
1134                          // >=:
1135                                  yyless(nchars-1);
1136                                yylvalstr = yylvalstr.substring(0,nchars-1);
1137                                                        if (yylvalstr.equalsIgnoreCase("||")){
1138                                  returni(concatenationop);
1139                              }else                                                             
1140                                  returni(cmpop);
1141                         }
1142                       
1143                         }
1144                                         else if ( (nchars >= 2) && (yylvalstr.charAt(0)== '|') && (yylvalstr.charAt(1) == '|'))
1145                        { //||--sss ,just get || only
1146                              yyless(2);
1147                              yylvalstr = yylvalstr.substring(0,2);
1148                              returni(concatenationop); //return as concatenationop, this is used in sybase only
1149                        }
1150                      else if ((nchars == 2) && ((yylvalstr.charAt(0) =='.') && ( (yylvalstr.charAt(1)=='*') || (yylvalstr.charAt(1) =='#')) ))
1151                          {
1152                            yyless(1);
1153                            returnc(yylvalstr.charAt(0));
1154                          }
1155                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '#')) ))
1156                          {
1157                            yyless(1);
1158                            returnc(yylvalstr.charAt(0));
1159                          }
1160                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '?')) ))
1161                          {
1162                            yyless(1);
1163                            returnc(yylvalstr.charAt(0));
1164                          }
1165                                        else if ( (nchars >= 3) && (yylvalstr.charAt(nchars-1) == '$' ) && (yylvalstr.charAt(nchars-2) == '$' ))
1166                                        { // =$$abc
1167                                            if (nchars == 3){
1168                                                        yyless(1);
1169                                                        returnc(yylvalstr.charAt(0));
1170                                                }else{
1171                                                        yyless(nchars-2);
1172                                                        yylvalstr = yylvalstr.substring(0,nchars-2);
1173                                                        returni(cmpop);
1174                                                }
1175                                        }
1176                                                  else if (((nchars > 2) && (yylvalstr.charAt(0) == '*'))
1177                              && (yylvalstr.charAt(1) == '/')
1178                              && (getyysstate() == xc)
1179                              )
1180                           { //in comment, and find */ , then it must the end of comment
1181                             yyless(2);
1182                                addlit(yylvalstr,yytextlen);
1183                             if (xcdepth <= 0)
1184                               {
1185                                 start(init);
1186                                 yylvalstr = litbufdup();
1187                                 returni(cmtslashstar);
1188                               }
1189                             else
1190                               xcdepth--;
1191                           }
1192                      else
1193                        returni(op);
1194               }
1195
1196          break;
1197          }
1198
1199  case 36:
1200                        
1201          {
1202               returni(iconst);
1203               break;
1204          }
1205
1206  case 37:
1207                        
1208          {
1209            ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
1210            nchars = yytextlen;
1211            if (yylvalstr.charAt(nchars-1) == '.')
1212              {
1213                dummych1 = get_char();
1214                unget_char(dummych1);
1215                if (dummych1 == '.')
1216                  {
1217                      yyless(nchars-1);
1218                    yylvalstr = yylvalstr.substring(0,nchars - 1);
1219                    returni (iconst);
1220                    return;//exit;
1221                  }
1222              }
1223            returni (fconst);
1224           break;
1225          }
1226
1227  case 38:
1228                        
1229          {
1230           returni (fconst);
1231           break;
1232          }
1233
1234  case 39:
1235              
1236          {
1237            int rw;
1238            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1239            else
1240               returni(ident);
1241            break;
1242          }
1243
1244
1245  case 40:
1246           
1247        {
1248                if (yylvalstr.charAt(0) == '$'){
1249                        returni(ident);
1250                }else
1251                        returni(variable);
1252
1253                break;
1254        }
1255                  
1256  case 41:
1257                  
1258          {
1259            if ((yylvalstr.charAt(0) == '$')&&(yylvalstr.charAt(1) == '_')){
1260              returni(ident);
1261            }else
1262              returni(variable);
1263
1264            break;
1265          }
1266 
1267  case 42:
1268           
1269          {
1270            returni(mslabel);
1271            break;
1272          }
1273
1274                  
1275  case 43:
1276               
1277          {
1278            returni(lexspace);
1279            break;
1280          }               
1281                  
1282  case 44:
1283                    
1284          {
1285            returni(UNICODE_ENCODE_ID);
1286            break;
1287          }               
1288                  
1289    
1290  case 45:
1291                        
1292          {
1293            returni( error);
1294            break;
1295          }
1296    default:{
1297     System.out.println("fatal error in yyaction");
1298    }
1299   }//switch
1300}/*yyaction*/;
1301
1302
1303
1304        }