Source code

001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************}
004{   Lexical analizer for GSQLParser component java version    }
005{   Copyright (c) 2004-2025 by Gudu software              }
006{****************************************************/
007
008package gudusoft.gsqlparser;
009
010
011import java.util.*;
012import java.io.InputStreamReader;
013
014import java.io.BufferedReader;
015import java.io.IOException;
016
017import gudusoft.gsqlparser.stmt.teradata.utilities.*;
018
019public class TLexerTeradata extends TCustomLexer{
020    static int yynmarks = 0  ;
021    static int yynmatches ;
022    static int yyntrans   ;
023    static int yynstates  ;
024    static int[]  yyk,yym ; // 1 based
025    static int[]  yytint;  // 1 based
026    static TYytRec[] yyt ;  // 1 based
027    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
028    private  static  String[] keywordlist;
029    static String   table_file;
030    static HashMap<String, Integer> keywordValueList;
031    static HashMap<Integer, Integer> keywordTypeList;
032          static String  bteqCmdStr,multiLoadCmdStr,fastExportCmdStr,fastLoadCmdStr;
033    static List<String> bteqCmdList,multiLoadCmdList,fastExportCmdList,fastLoadCmdList;
034        static int[][] yystateTable;
035        static HashMap<String, Integer> operatorList;
036
037    static {
038                        table_file = "/gudusoft/gsqlparser/parser/teradata/teradata_lex_table.txt";
039      keywordValueList = new HashMap<String, Integer>();
040              keywordTypeList = new HashMap<Integer, Integer>();
041                        bteqCmdList = new ArrayList<String>( );
042                        multiLoadCmdList = new ArrayList<String>( );
043                        fastExportCmdList = new ArrayList<String>( );
044                        fastLoadCmdList = new ArrayList<String>( );
045
046                        // Initialize operator list for proper tokenization of operators followed by +/-
047                        operatorList = new HashMap<String, Integer>();
048                        operatorList.put("+", Integer.valueOf('+'));
049                        operatorList.put("-", Integer.valueOf('-'));
050                        operatorList.put("*", Integer.valueOf('*'));
051                        operatorList.put("/", Integer.valueOf('/'));
052                        operatorList.put("%", Integer.valueOf('%'));
053                        operatorList.put("^", Integer.valueOf('^'));
054                        operatorList.put("&", Integer.valueOf('&'));
055                        operatorList.put("|", Integer.valueOf('|'));
056                        operatorList.put("~", Integer.valueOf('~'));
057                        operatorList.put("!", Integer.valueOf('!'));
058                        operatorList.put(">", Integer.valueOf('>'));
059                        operatorList.put("<", Integer.valueOf('<'));
060                        operatorList.put("=", Integer.valueOf('='));
061                        operatorList.put("<=", TBaseType.less_equal);
062                        operatorList.put("<>", TBaseType.not_equal);
063                        operatorList.put(">=", TBaseType.great_equal);
064                        operatorList.put("!=", TBaseType.not_equal);
065                        operatorList.put("^=", TBaseType.not_equal);
066                        operatorList.put("^<", TBaseType.not_less);
067                        operatorList.put("^>", TBaseType.not_great);
068                        operatorList.put("!<", TBaseType.not_less);   // SQL Server style not less than
069                        operatorList.put("!>", TBaseType.not_great);  // SQL Server style not greater than
070                        operatorList.put("**", TBaseType.exponentiate);
071                        operatorList.put("||", TBaseType.concatenationop);
072
073                        if (TBaseType.enterprise_edition||TBaseType.teradata_edition){
074                                inittable();
075                        }
076    }
077
078    public TLexerTeradata(){
079          super();
080          dbvendor = EDbVendor.dbvteradata;
081          sourcetokens = new TSourceTokenList();
082    }
083
084    public void setTeradataUtilityType(TeradataUtilityType teradataUtilityType) {
085        this.teradataUtilityType = teradataUtilityType;
086    }
087
088    private TeradataUtilityType teradataUtilityType = TeradataUtilityType.BTEQ;
089    
090    public TSourceTokenList sourcetokens;
091
092
093
094    int issystemvariable(String str){
095        return -1;// -1 means not a system variable
096    }
097
098public boolean canBeColumnName(int tokencode){
099    //http://blog.csdn.net/superbeck/article/details/5387476
100    boolean ret = false;
101    int modifiers = keyword_type_identifier | keyword_type_column ;
102    Integer s = keywordTypeList.get(tokencode);
103     if (s != null){
104        int modifier = s;
105        ret = (modifiers & modifier) == modifier;
106    }
107
108    return ret;
109}
110
111   public EFindSqlStateType cmdType(TSourceToken token){
112        
113        // use  isMultiLoadCmd,isFastExportCmd,isFastLoadCmd,isBteqCmd to find the command type,
114        String str = token.toString();
115        String str2 = null;
116        String str3 = null;
117        if (token.nextSolidToken() != null) {
118            str2 = token.nextSolidToken().toString();
119            if (token.nextSolidToken().nextSolidToken() != null) {
120              str3 = token.nextSolidToken().nextSolidToken().toString();
121            }
122        }
123
124    // First check the longest possible command (str + str2 + str3)
125    if (str3 != null) {
126        String combinedStr3 = str + " " + str2 + " " + str3;
127        int matchCount = 0;
128        boolean isMultiLoad3 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr3) != null;
129        boolean isFastExport3 = FastExportCmdType.searchFastExportCmd(combinedStr3) != null;
130        boolean isFastLoad3 = FastLoadCmdType.searchFastLoadCmd(combinedStr3) != null;
131        boolean isBteq3 = BteqCmdType.searchBteqCmd(combinedStr3) != null;
132
133        if (isMultiLoad3) matchCount++;
134        if (isFastExport3) matchCount++;
135        if (isFastLoad3) matchCount++;
136        if (isBteq3) matchCount++;
137
138        if (matchCount > 1) {
139            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq3) return EFindSqlStateType.stBTEQCmd;
140            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
141            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport3) return EFindSqlStateType.stFastExportCmd;
142            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
143
144            // throw new IllegalStateException("Ambiguous command '" + combinedStr3 + "' matches multiple utility types");
145            matchCount = 1;
146        }
147
148        if (matchCount == 1) {
149            if (isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
150            if (isFastExport3) return EFindSqlStateType.stFastExportCmd;
151            if (isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
152            if (isBteq3) return EFindSqlStateType.stBTEQCmd;
153        }
154    }
155
156    // Then check medium length command (str + str2)
157    if (str2 != null) {
158        String combinedStr2 = str + " " + str2;
159        int matchCount = 0;
160        boolean isMultiLoad2 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr2) != null;
161        boolean isFastExport2 = FastExportCmdType.searchFastExportCmd(combinedStr2) != null;
162        boolean isFastLoad2 = FastLoadCmdType.searchFastLoadCmd(combinedStr2) != null;
163        boolean isBteq2 = BteqCmdType.searchBteqCmd(combinedStr2) != null;
164
165        if (isMultiLoad2) matchCount++;
166        if (isFastExport2) matchCount++;
167        if (isFastLoad2) matchCount++;
168        if (isBteq2) matchCount++;
169
170        if (matchCount > 1) {
171            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq2) return EFindSqlStateType.stBTEQCmd;
172            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
173            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport2) return EFindSqlStateType.stFastExportCmd;
174            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
175
176            //throw new IllegalStateException("Ambiguous command '" + combinedStr2 + "' matches multiple utility types");
177            matchCount = 1;
178        }
179        if (matchCount == 1) {
180            if (isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
181            if (isFastExport2) return EFindSqlStateType.stFastExportCmd;
182            if (isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
183            if (isBteq2) return EFindSqlStateType.stBTEQCmd;
184        }
185    }
186
187    // Finally check single word command (str)
188    boolean isMultiLoad = MultiLoadCmdType.searchMultiLoadCmd(str) != null;
189    boolean isFastExport = FastExportCmdType.searchFastExportCmd(str) != null;
190    boolean isFastLoad = FastLoadCmdType.searchFastLoadCmd(str) != null;
191    boolean isBteq = BteqCmdType.searchBteqCmd(str) != null;
192    int matchCount = 0;
193    if (isMultiLoad) matchCount++;
194    if (isFastExport) matchCount++;
195    if (isFastLoad) matchCount++;
196    if (isBteq) matchCount++;
197
198    if (matchCount > 1) {
199        if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq) return EFindSqlStateType.stBTEQCmd;
200        if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
201        if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport) return EFindSqlStateType.stFastExportCmd;
202        if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad) return EFindSqlStateType.stFastLoadCmd;
203
204       // throw new IllegalStateException("Ambiguous command '" + str + "' matches multiple utility types");
205        matchCount = 1;
206    }
207
208    if (matchCount == 1) {
209      if (isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
210      if (isFastExport) return EFindSqlStateType.stFastExportCmd;
211      if (isFastLoad) return EFindSqlStateType.stFastLoadCmd;
212      if (isBteq) return EFindSqlStateType.stBTEQCmd;
213  }
214    return EFindSqlStateType.stnormal;
215
216  }
217
218
219
220
221 public  int iskeyword(String str){
222        int ret = -1;
223        Integer s = keywordValueList.get(str.toUpperCase(Locale.ENGLISH));
224        if( s != null){
225            ret = s;
226        }
227        return ret;// -1 means not a keyword
228     }
229         
230    public static EKeywordType getKeywordType(String keyword){
231        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
232    }            
233
234     public int getkeywordvalue(String keyword){
235        int ret = 0;
236        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
237        if( s != null){
238            ret = s;
239         }
240        return ret;// 0 means not a keyword
241     }
242
243    static void yystateLookupConfigure() {
244        int yystates = yytl.length;
245        yystateTable = new int[257][yystates];
246
247        // initialize to empty
248        for(int i = 0; i < yystates; i++) {
249            for (int j = 0; j < 257; j++)
250                yystateTable[j][i] = -1;
251        }
252
253        for(int i = 0; i < yystates; i++) {
254            int low = yytl[i];
255            int high = yyth[i];
256            for (int j = low; j <= high; j++) {
257                for (char c: yyt[j].cc) {
258                    yystateTable[c][i] = j;
259                }
260            }
261        }
262    }
263        
264    int yylex(){
265          int yyn;
266           while (true) { // top level while
267              yynew();
268              while (true){  //scan
269                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
270                     yymark(yyk[yyn]);
271                 }
272
273                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
274                    yymatch(yym[yyn]);
275                 }
276
277                 if(yytl[yystate] > yyth[yystate]){
278                     break;
279                 }
280
281                 yyscan();
282//                 yyn = yytl[yystate];
283                 totablechar();
284//                 while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
285//                   yyn++;
286//                 }
287//                 if (yyn > yyth[yystate]){
288//                     break;
289//                 }
290
291                 yyn = yystateTable[yytablechar][yystate];
292                 if (yyn == -1)
293                     break;
294                                         
295                 yystate = yyt[yyn].s;
296              } //scan
297
298              while (true){ //action
299                int yyrule;
300                if ( (yyrule = yyfind()) != -1 ){
301                   yyaction(yyrule);
302                   if (yyreject){
303                       continue;
304                   }
305                }else if( (!yydefault() ) && (yywrap()) ){
306                   yyclear();
307                   returni(0);
308                }
309                break;
310              }
311
312              if (!yydone) {
313                  continue;
314              }
315              break;
316            } // top level while
317
318           return yyretval;
319        }
320
321    
322
323    static void inittable(){
324                
325                //if (yynmarks > 0) return; //init table already
326
327        String line;
328        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
329        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
330        int c=0;
331        keywordValueList.clear();
332        keywordTypeList.clear();
333        
334        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerTeradata.class.getResourceAsStream(table_file)));
335
336            try{
337                while( (line = br.readLine()) != null){
338                          if (line.trim().startsWith("yynmarks=")){
339                             String[] ss = line.split("[=;]");
340                              yynmarks=Integer.parseInt(ss[1].trim());
341                              yyk = new int[yynmarks+1];
342                          }else if (line.trim().startsWith("yynmatches=")){
343                              String[] ss = line.split("[=;]");
344                               yynmatches=Integer.parseInt(ss[1].trim());
345                               yym = new int[yynmatches+1];
346                          }else if (line.trim().startsWith("yyntrans=")){
347                              String[] ss = line.split("[=;]");
348                               yyntrans=Integer.parseInt(ss[1].trim());
349                               yytint = new int[yyntrans+1];
350                               yyt = new TYytRec[yyntrans+1];
351                          }else if (line.trim().startsWith("yynstates=")){
352                              String[] ss = line.split("[=;]");
353                               yynstates=Integer.parseInt(ss[1].trim());
354                               yykl = new int[yynstates];
355                               yykh = new int[yynstates];
356                              yyml = new int[yynstates];
357                              yymh = new int[yynstates];
358                              yytl = new int[yynstates];
359                              yyth = new int[yynstates];
360                          }else if (line.trim().startsWith("<end>")){
361                              if (inyyk){
362                                  inyyk = false;
363                                 if (yynmarks+1 != yyk_count ){
364                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
365                                 }
366                              }
367                              else if(inyym){
368                                     inyym = false;
369                                    if (yynmatches+1 != yym_count ){
370                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
371                                    }
372                              }
373                              else if(inyykl){
374                                     inyykl = false;
375                                    if (yynstates != yykl_count ){
376                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
377                                    }
378                              }
379                              else if(inyykh){
380                                     inyykh = false;
381                                    if (yynstates != yykh_count ){
382                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
383                                    }
384                              }
385                              else if(inyyml){
386                                     inyyml = false;
387                                    if (yynstates != yyml_count ){
388                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
389                                    }
390                              }
391                              else if(inyymh){
392                                     inyymh = false;
393                                    if (yynstates != yymh_count ){
394                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
395                                    }
396                              }
397                              else if(inyytl){
398                                     inyytl = false;
399                                    if (yynstates != yytl_count ){
400                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
401                                    }
402                              }
403                              else if(inyyth){
404                                     inyyth = false;
405                                    if (yynstates != yyth_count ){
406                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
407                                    }
408                              }
409                              else if(inyytint){
410                                     inyytint = false;
411                                    if (yyntrans + 1 != yytint_count ){
412                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
413                                    }
414                              }
415                              else if(inyyt){
416                                     inyyt = false;
417                                    if (yyntrans+1 != yyt_count ){
418                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
419                                    }
420                              }
421                              else if(inkeyword){
422                                     inkeyword = false;
423                              }
424                          }else if(line.trim().startsWith("yyk =")){
425                             inyyk = true; 
426                          }else if(line.trim().startsWith("yym =")){
427                             inyym = true;
428                          }else if(line.trim().startsWith("yykl =")){
429                             inyykl = true;
430                          }else if(line.trim().startsWith("yykh =")){
431                             inyykh = true;
432                          }else if(line.trim().startsWith("yyml =")){
433                             inyyml = true;
434                          }else if(line.trim().startsWith("yymh =")){
435                             inyymh = true;
436                          }else if(line.trim().startsWith("yytl =")){
437                             inyytl = true;
438                          }else if(line.trim().startsWith("yyth =")){
439                             inyyth = true;
440                          }else if(line.trim().startsWith("yytint =")){
441                             inyytint = true;
442                          }else if(line.trim().startsWith("yyt =")){
443                             inyyt = true;
444                        }else if(line.trim().startsWith("keywordsvalue =")){
445                           inkeyword = true;
446                        }else if(inyyk){
447                             String[] ss = line.split("[,]");
448                               for(int j=0;j<ss.length;j++){
449                                   // System.out.println(ss[j].trim());
450                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
451                               }
452                          }else if(inyym){
453                               String[] ss = line.split("[,]");
454                                 for(int j=0;j<ss.length;j++){
455                                     // System.out.println(ss[j].trim());
456                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
457                                 }
458                          }else if(inyykl){
459                               String[] ss = line.split("[,]");
460                                 for(int j=0;j<ss.length;j++){
461                                    //  System.out.println(ss[j].trim());
462                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
463                                 }
464                          }else if(inyykh){
465                               String[] ss = line.split("[,]");
466                                 for(int j=0;j<ss.length;j++){
467                                     // System.out.println(ss[j].trim());
468                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
469                                 }
470                          }else if(inyyml){
471                               String[] ss = line.split("[,]");
472                                 for(int j=0;j<ss.length;j++){
473                                     // System.out.println(ss[j].trim());
474                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
475                                 }
476                          }else if(inyymh){
477                               String[] ss = line.split("[,]");
478                                 for(int j=0;j<ss.length;j++){
479                                     // System.out.println(ss[j].trim());
480                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
481                                 }
482                          }else if(inyytl){
483                               String[] ss = line.split("[,]");
484                                 for(int j=0;j<ss.length;j++){
485                                     // System.out.println(ss[j].trim());
486                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
487                                 }
488                          }else if(inyyth){
489                               String[] ss = line.split("[,]");
490                                 for(int j=0;j<ss.length;j++){
491                                     // System.out.println(ss[j].trim());
492                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
493                                 }
494                          }else if(inyytint){
495                               String[] ss = line.split("[,]");
496                                 for(int j=0;j<ss.length;j++){
497                                     // System.out.println(ss[j].trim());
498                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
499                                 }
500                          }else if(inyyt){
501                                //System.out.println(line.trim());
502
503                              c = 0;
504                              String[] st = line.trim().split(",,");
505                              char[] tmp = new char[st.length];
506                              for(int i=0;i<st.length;i++){
507
508                                  if(st[i].startsWith("\'")) {
509                                      if(st[i].length() == 3){  // 'a'
510                                          tmp[c++] = st[i].charAt(1);
511                                      }else if(st[i].length() == 4) { // '\\'
512                                          tmp[c++] = st[i].charAt(2);
513                                      }else{
514                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
515                                      }
516                                  }else{
517                                      try{
518                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
519                                          } catch (NumberFormatException nfe) {
520                                             System.out.println("NumberFormatException: " + nfe.getMessage());
521                                          }
522                                  }
523                              } //while hasmoreTokens
524
525                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
526                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
527                              yyt_count++;
528
529                          }else if(inkeyword){
530                              String[] ss =line.split("[=]");
531
532                              int val1 = -1;
533                              int val2 = -1;
534                              try {
535                                  val1 = Integer.parseInt(ss[1]);
536                                  val2 = Integer.parseInt(ss[2]);
537                              }
538                              catch (NumberFormatException nfe) {
539                                  System.out.println("NumberFormatException: " + nfe.getMessage());
540                              }
541                              keywordValueList.put(ss[0].toUpperCase(),val1);
542                              keywordTypeList.put(val1,val2);
543                             }
544                }
545                }catch(IOException e){
546                  System.out.println(e.toString());
547                }
548                                
549                                yystateLookupConfigure();
550
551    }
552
553
554
555    void yyaction(int yyruleno){
556
557
558
559      int ic;
560      char[] tmparray = {'=','+','-','*','/','>','<'};
561
562      yylvalstr = getyytext();
563  /* actions: */
564  switch(yyruleno){
565  case 1:
566                
567        {
568              addlit(yylvalstr,yytextlen);
569              if (xcdepth <= 0)
570                 {
571                  start(init);
572                  yylvalstr = litbufdup();
573                  returni(cmtslashstar);
574                 }
575              else
576                 xcdepth--;
577
578            break;
579        }
580
581
582  case 2:
583                
584          {
585             // xcdepth++;
586              yyless(1);
587              addlit(yylvalstr,1);
588              break;
589           }
590
591  case 3:
592                        
593          {
594
595              if (getyysstate() == xq)
596              {
597                  nchars = yytextlen;
598                  addlit(yylvalstr, yytextlen-1);
599                  yyless(nchars-1);
600                  return;//exit;
601              }
602
603              xcdepth = 0;
604              start(xc);
605              startlit();
606              yyless(2);
607              addlit(yylvalstr,yytextlen);
608
609          break;
610          }
611
612  case 4:
613                
614          {
615            addlit(yylvalstr,yytextlen);
616            break;
617          }
618
619  case 5:
620                
621          {
622              addlitchar(yylvalstr.charAt(0));
623              break;
624          }
625
626                  
627  case 6:
628                       
629        {
630                                        start(init);
631                                        addlit(yylvalstr, yytextlen);
632                                        yylvalstr = litbufdup();
633                                        returni(ident);
634                break;
635        }
636
637  case 7:
638                        
639        {
640          if ((getyysstate() == xd)||(getyysstate() == xq))
641          {
642              addlit(yylvalstr, 1);
643              yyless(1);
644              return;//exit;
645          }     
646                                        //token_start := yylvalStr;
647                                        start(xbacktick);
648                                        startlit();
649                                        addlit(yylvalstr, yytextlen);
650                break;
651        }
652
653
654  case 8:
655                       
656        {
657                addlit(yylvalstr, yytextlen);
658                break;
659  }
660
661  case 9:
662                       
663        {
664    addlit(yylvalstr, yytextlen);
665                break;
666   }
667                  
668
669  case 10:
670                  
671          {
672            returni(ident);
673            break;
674          }
675          
676  case 11:
677                           
678                        {
679                                if ((getyysstate() == xq)){
680                      addlit(yylvalstr, 1);
681                      yyless(1);
682                      return;
683                                }else{
684                                        returni(sconst);
685                                        break;
686                                }
687                        }
688                        
689  case 12:
690                           
691                        {
692                                if ((getyysstate() == xq)){
693                      addlit(yylvalstr, 1);
694                      yyless(1);
695                      return;
696                                }else{
697                                        returni(TBaseType.teradata_unicode_sconst);
698                                        break;
699                                }
700                        }
701                        
702                        
703  case 13:
704                
705                        {
706                                if ((getyysstate() == xq)){
707                      addlit(yylvalstr, 1);
708                      yyless(1);
709                      return;
710                                }else{
711                                        returni(sconst);
712                                        break;
713                                }
714                        }
715                        
716  case 14:
717                                        
718                        {
719                        returni(sconst);
720                        break;
721                        }                       
722                        
723  case 15:
724                
725                        {
726                        returni(not_equal);
727                        break;
728                        }                       
729
730  case 16:
731                
732          {
733              start(init);
734              addlit(yylvalstr, yytextlen);
735              yylvalstr = litbufdup();
736              returni(sconst);
737              break;
738          }
739
740  case 17:
741                           
742          {
743              start(init);
744              addlit(yylvalstr, yytextlen);
745              yylvalstr = litbufdup();
746              returni(sconst);
747              break;
748          }
749
750  case 18:
751                
752          {
753            addlit(yylvalstr, yytextlen);
754            break;
755          }
756
757
758  case 19:
759                        
760          {
761              if (yysstate == xd)
762                {
763                  addlit(yylvalstr, yytextlen);
764                }
765              else
766                {
767                  start(xq);
768                  startlit();
769                  addlit(yylvalstr, yytextlen);
770                  dummych1 = get_char();
771                  if (dummych1 == '\\') // recognize string like '\'
772                    {
773                      dummych2 = get_char();
774                      if (dummych2 == '\'')
775                      {
776                          start(init);
777                          addlit("\\", 1);
778                          addlit("\'", 1);
779                          yylvalstr = litbufdup();
780                          returni(sconst);
781                      }
782                      else
783                       {
784                          unget_char(dummych2);
785                          unget_char(dummych1);
786                        }
787                    }
788                  else
789                    unget_char(dummych1);
790                }
791
792              break;
793          }
794
795  case 20:
796                
797          {
798              addlit(yylvalstr, yytextlen);
799              break;
800          }
801
802  case 21:
803                
804          {
805            dummych1 = get_char();
806            unget_char(dummych1);
807            if (dummych1 == (char)10)
808              {
809                dummych1 = get_char();
810                addlit(yylvalstr+dummych1, yytextlen+1);
811              } else if (dummych1 == '\\')
812                  {
813                      dummych1 = get_char();
814                      dummych2 = get_char();
815                      if (dummych2 == '\'')
816                        {
817                          unget_char(dummych2);
818                          addlit(yylvalstr+dummych1, yytextlen+1);
819                        }
820                      else
821                        addlit(yylvalstr+dummych1+dummych2, yytextlen+2);
822                  }
823                else
824                  addlit(yylvalstr, yytextlen);
825
826          break;
827         }
828        
829  case 22:
830                 
831          {
832              addlit(yylvalstr, yytextlen);
833              break;
834          }
835
836
837
838  case 23:
839                
840          {
841              start(init);
842              addlit(yylvalstr, yytextlen);
843              if (literallen == 0)   returni (error);
844              if (literallen >= namedatalen)
845              {
846                 setlengthofliteralbuf(namedatalen);
847                 literallen = namedatalen;
848              }
849              yylvalstr = litbufdup();
850              returni (ident);
851
852              break;
853          }
854
855  case 24:
856               
857          {
858          addlit(yylvalstr, yytextlen);
859          break;
860          }
861
862  case 25:
863                        
864          {
865              start(xd);
866              startlit();
867              addlit(yylvalstr, yytextlen);
868              break;
869          }
870        
871  case 26:
872                
873        {
874                
875                dummych1 = get_char();
876                unget_char(dummych1);
877                if (dummych1 == (char)10)
878                  {
879                dummych1 = get_char();
880                addlit(yylvalstr+dummych1, yytextlen+1);
881                  } else
882                    addlit(yylvalstr, yytextlen);
883                
884                break;
885        }
886
887
888  case 27:
889                
890          {
891            returni(lexnewline);
892            break;
893          }
894
895  case 28:
896        
897          {
898              returni(lexspace);
899              break;
900          }
901
902  case 29:
903                
904          {
905          if ((getyysstate() == xq)
906              || (getyysstate() == xd)
907              || (getyysstate() == xc)
908              )
909          {
910              addlit(yylvalstr, 1);
911              yyless(1);
912              return;//exit;
913          }
914
915          returni(cmtdoublehyphen);
916          break;
917          }
918
919  case 30:
920                 
921          {
922               returni(ident);
923               break;
924          }
925
926  case 31:
927      
928          {
929               returnc(yylvalstr.charAt(0));
930               break;
931          }
932
933  case 32:
934                       
935          {
936            returni(cmpop);
937            break;
938          }
939        
940  case 33:
941                  
942      {
943        returni(concatenationop);
944        break;
945      }
946          
947  case 34:
948           
949          {
950
951          if (getyysstate() == xc)
952             {
953              slashstar = yylvalstr.indexOf("*/");
954                if (slashstar >= 0)
955                  {
956                      addlit(yylvalstr,slashstar+2);
957                      yylvalstr = litbufdup();
958                      yyless(slashstar+2);
959
960                                                                                        if (xcdepth <= 0){
961                                                                                                start(init);
962                                                                                                returni(cmtslashstar);
963                                                                                        }else{
964                                                                                                xcdepth--;
965                                                                                        }
966                  }
967                else
968                  {
969                      addlit(yylvalstr,1);
970                      yyless(1);
971                  }
972             }
973          else
974            {
975              nchars = yytextlen;
976              slashstar = yylvalstr.indexOf("/*");
977              dashdash = yylvalstr.indexOf("--");
978              if ((slashstar >= 0) && (dashdash >= 0))
979                {
980                  //if both appear, take the first one
981                   if (slashstar > dashdash)
982                    {slashstar = dashdash;}
983                }
984              else
985                {
986                  //   if slashstar=0 then slashstar := dashdash;
987                  // add (getyysstate <> xc) to avoid something like this */--,here */ should be handled instead of --
988                  if ((slashstar > 0) && (getyysstate() != xc)) {
989                    nchars = slashstar;
990                    }
991                }
992
993              // Search for known operators from longest to shortest
994              // This handles cases like <=-.01 where <= should be returned, not <=-
995              boolean searchOp = false;
996              while (nchars > 0){
997                  String opStr = yylvalstr.substring(0,nchars);
998                  Integer opCode = operatorList.get(opStr);
999                  if (opCode != null){
1000                      if (nchars < yytextlen){
1001                          yyless(nchars);
1002                          yylvalstr = yylvalstr.substring(0,nchars);
1003                      }
1004                      searchOp = true;
1005                      returni(opCode);
1006                      break;
1007                  }
1008                  nchars--;
1009              }
1010
1011              if (searchOp) break;
1012
1013              nchars = yytextlen; // reset nchars
1014
1015              while ((nchars > 1)
1016                      && ( (yylvalstr.charAt(nchars-1) == '+' )
1017                          || (yylvalstr.charAt(nchars-1) =='-'))
1018                      && (getyysstate() != xc))
1019                {
1020                  for (ic = nchars - 1; ic>=1; ic--)
1021                  {
1022                    if (isopchar(yylvalstr.charAt(ic-1)))  break;
1023                  }
1024                  if (ic >= 1) break;
1025                  nchars--;
1026                }
1027
1028              if (nchars < yytextlen)
1029                {
1030                  //Strip the unwanted chars from the token
1031                  yyless(nchars);
1032                  yylvalstr = yylvalstr.substring(0,nchars);
1033                }
1034
1035                  ///*
1036                  // * If what we have left is only one char, and it's
1037                  // * one of the characters matching "self", then
1038                  // * return it as a character token the same way
1039                  // * that the "self" rule would have.
1040                  // * make sure @ return as self char, by james wang
1041                  // */
1042                  if ((nchars == 1) && (isselfchar(yylvalstr.charAt(0)) || (yylvalstr.charAt(0) == '@')))
1043                    {
1044                      returnc(yylvalstr.charAt(0));
1045                    }
1046                  else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '.') )
1047                    {
1048                        yyless(1);
1049                        returnc(yylvalstr.charAt(0));
1050                    }
1051                  else if (
1052                              (nchars >= 2)
1053                                  &&(
1054                                       charinarray(yylvalstr.charAt(0), tmparray)
1055                                       && (yylvalstr.charAt(1) == ':')
1056                                       )
1057                           )
1058                      {
1059                        yyless(nchars-1);
1060                         yylvalstr = yylvalstr.substring(0,nchars-1);
1061                        if (nchars == 2)
1062                          returnc(yylvalstr.charAt(0));
1063                        else
1064                          returni(op);
1065                      }
1066                    else if (
1067                                (nchars >= 2)
1068                                    && (
1069                                          charinarray(yylvalstr.charAt(nchars-1-1),tmparray)
1070                                          && (yylvalstr.charAt(nchars-1) == '.')
1071                                         )
1072                               )
1073                        {
1074                          yyless(nchars-1);
1075                           yylvalstr = yylvalstr.substring(0,nchars-1);
1076                          if (nchars == 2)
1077                            returnc(yylvalstr.charAt(0));
1078                          else
1079                            returni(op);
1080                        }
1081                    else if ( (nchars == 2) && ((yylvalstr.charAt(nchars-1) == '$')||(yylvalstr.charAt(nchars-1) == '?')) )
1082                      {
1083                        yyless(nchars-1);
1084                        yylvalstr = yylvalstr.substring(0,nchars-1);
1085                        returnc(yylvalstr.charAt(0));
1086                      }
1087                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == '@') )
1088                      {
1089                        if (nchars == 2)
1090                          { // such as >@
1091                              yyless(nchars-1);
1092                            yylvalstr = yylvalstr.substring(0,nchars-1);
1093                             returnc(yylvalstr.charAt(0));
1094                          }
1095                        else
1096                          {
1097                            if (yylvalstr.charAt(nchars-1-1) == '@')
1098                              { //such as >@@,>=@@
1099                                  yyless(nchars-2);
1100                                yylvalstr = yylvalstr.substring(0,nchars-2);
1101                                if (nchars == 3)
1102                                    returnc(yylvalstr.charAt(0));
1103                                else
1104                                    returni(cmpop);
1105                              }
1106                            else
1107                              { // >=@
1108                                  yyless(nchars-1);
1109                                yylvalstr = yylvalstr.substring(0,nchars-1);
1110                                  returni(cmpop);
1111                              }
1112                          }
1113                         }
1114                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == ':') )
1115                      {
1116                        if (nchars == 2)
1117                          { // such as >:
1118                            yyless(nchars-1);
1119                            yylvalstr = yylvalstr.substring(0,nchars-1);
1120                            returnc(yylvalstr.charAt(0));
1121                          }
1122                        else
1123                          {
1124                          // >=:
1125                                  yyless(nchars-1);
1126                                yylvalstr = yylvalstr.substring(0,nchars-1);
1127                                                        if (yylvalstr.equalsIgnoreCase("||")){
1128                                  returni(concatenationop);
1129                              }else                                                             
1130                                  returni(cmpop);
1131                         }
1132                       
1133                         }
1134                                         else if ( (nchars >= 2) && (yylvalstr.charAt(0)== '|') && (yylvalstr.charAt(1) == '|'))
1135                        { //||--sss ,just get || only
1136                              yyless(2);
1137                              yylvalstr = yylvalstr.substring(0,2);
1138                              returni(concatenationop); //return as concatenationop, this is used in sybase only
1139                        }
1140                      else if ((nchars == 2) && ((yylvalstr.charAt(0) =='.') && ( (yylvalstr.charAt(1)=='*') || (yylvalstr.charAt(1) =='#')) ))
1141                          {
1142                            yyless(1);
1143                            returnc(yylvalstr.charAt(0));
1144                          }
1145                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '#')) ))
1146                          {
1147                            yyless(1);
1148                            returnc(yylvalstr.charAt(0));
1149                          }
1150                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '?')) ))
1151                          {
1152                            yyless(1);
1153                            returnc(yylvalstr.charAt(0));
1154                          }
1155                                        else if ( (nchars >= 3) && (yylvalstr.charAt(nchars-1) == '$' ) && (yylvalstr.charAt(nchars-2) == '$' ))
1156                                        { // =$$abc
1157                                            if (nchars == 3){
1158                                                        yyless(1);
1159                                                        returnc(yylvalstr.charAt(0));
1160                                                }else{
1161                                                        yyless(nchars-2);
1162                                                        yylvalstr = yylvalstr.substring(0,nchars-2);
1163                                                        returni(cmpop);
1164                                                }
1165                                        }
1166                                                  else if (((nchars > 2) && (yylvalstr.charAt(0) == '*'))
1167                              && (yylvalstr.charAt(1) == '/')
1168                              && (getyysstate() == xc)
1169                              )
1170                           { //in comment, and find */ , then it must the end of comment
1171                             yyless(2);
1172                                addlit(yylvalstr,yytextlen);
1173                             if (xcdepth <= 0)
1174                               {
1175                                 start(init);
1176                                 yylvalstr = litbufdup();
1177                                 returni(cmtslashstar);
1178                               }
1179                             else
1180                               xcdepth--;
1181                           }
1182                      else
1183                        returni(op);
1184               }
1185
1186          break;
1187          }
1188
1189  case 35:
1190                        
1191          {
1192               returni(iconst);
1193               break;
1194          }
1195
1196  case 36:
1197                        
1198          {
1199            ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
1200            nchars = yytextlen;
1201            if (yylvalstr.charAt(nchars-1) == '.')
1202              {
1203                dummych1 = get_char();
1204                unget_char(dummych1);
1205                if (dummych1 == '.')
1206                  {
1207                      yyless(nchars-1);
1208                    yylvalstr = yylvalstr.substring(0,nchars - 1);
1209                    returni (iconst);
1210                    return;//exit;
1211                  }
1212              }
1213            returni (fconst);
1214           break;
1215          }
1216
1217  case 37:
1218                        
1219          {
1220           returni (fconst);
1221           break;
1222          }
1223
1224  case 38:
1225              
1226          {
1227            int rw;
1228            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1229            else
1230               returni(ident);
1231            break;
1232          }
1233
1234
1235  case 39:
1236           
1237        {
1238                if (yylvalstr.charAt(0) == '$'){
1239                        returni(ident);
1240                }else
1241                        returni(variable);
1242
1243                break;
1244        }
1245                  
1246  case 40:
1247                  
1248          {
1249            if ((yylvalstr.charAt(0) == '$')&&(yylvalstr.charAt(1) == '_')){
1250              returni(ident);
1251            }else
1252              returni(variable);
1253
1254            break;
1255          }
1256 
1257  case 41:
1258           
1259          {
1260            returni(mslabel);
1261            break;
1262          }
1263
1264                  
1265  case 42:
1266               
1267          {
1268            returni(lexspace);
1269            break;
1270          }               
1271                  
1272  case 43:
1273                    
1274          {
1275            returni(UNICODE_ENCODE_ID);
1276            break;
1277          }               
1278                  
1279    
1280  case 44:
1281                        
1282          {
1283            returni( error);
1284            break;
1285          }
1286    default:{
1287     System.out.println("fatal error in yyaction");
1288    }
1289   }//switch
1290}/*yyaction*/;
1291
1292
1293
1294        }