Source code

001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************}
004{   Lexical analizer for GSQLParser component java version    }
005{   Copyright (c) 2004-2025 by Gudu software              }
006{****************************************************/
007
008package gudusoft.gsqlparser;
009
010
011import java.util.*;
012import java.io.InputStreamReader;
013
014import java.io.BufferedReader;
015import java.io.IOException;
016
017import gudusoft.gsqlparser.stmt.teradata.utilities.*;
018
019public class TLexerTeradata extends TCustomLexer{
020    static int yynmarks = 0  ;
021    static int yynmatches ;
022    static int yyntrans   ;
023    static int yynstates  ;
024    static int[]  yyk,yym ; // 1 based
025    static int[]  yytint;  // 1 based
026    static TYytRec[] yyt ;  // 1 based
027    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
028    private  static  String[] keywordlist;
029    static String   table_file;
030    static HashMap<String, Integer> keywordValueList;
031    static HashMap<Integer, Integer> keywordTypeList;
032          static String  bteqCmdStr,multiLoadCmdStr,fastExportCmdStr,fastLoadCmdStr;
033    static List<String> bteqCmdList,multiLoadCmdList,fastExportCmdList,fastLoadCmdList;
034        static int[][] yystateTable;
035        static HashMap<String, Integer> operatorList;
036
037    static {
038                        table_file = "/gudusoft/gsqlparser/parser/teradata/teradata_lex_table.txt";
039      keywordValueList = new HashMap<String, Integer>();
040              keywordTypeList = new HashMap<Integer, Integer>();
041                        bteqCmdList = new ArrayList<String>( );
042                        multiLoadCmdList = new ArrayList<String>( );
043                        fastExportCmdList = new ArrayList<String>( );
044                        fastLoadCmdList = new ArrayList<String>( );
045
046                        // Initialize operator list for proper tokenization of operators followed by +/-
047                        operatorList = new HashMap<String, Integer>();
048                        operatorList.put("+", Integer.valueOf('+'));
049                        operatorList.put("-", Integer.valueOf('-'));
050                        operatorList.put("*", Integer.valueOf('*'));
051                        operatorList.put("/", Integer.valueOf('/'));
052                        operatorList.put("%", Integer.valueOf('%'));
053                        operatorList.put("^", Integer.valueOf('^'));
054                        operatorList.put("&", Integer.valueOf('&'));
055                        operatorList.put("|", Integer.valueOf('|'));
056                        operatorList.put("~", Integer.valueOf('~'));
057                        operatorList.put("!", Integer.valueOf('!'));
058                        operatorList.put(">", Integer.valueOf('>'));
059                        operatorList.put("<", Integer.valueOf('<'));
060                        operatorList.put("=", Integer.valueOf('='));
061                        operatorList.put("<=", TBaseType.less_equal);
062                        operatorList.put("<>", TBaseType.not_equal);
063                        operatorList.put(">=", TBaseType.great_equal);
064                        operatorList.put("!=", TBaseType.not_equal);
065                        operatorList.put("^=", TBaseType.not_equal);
066                        operatorList.put("^<", TBaseType.not_less);
067                        operatorList.put("^>", TBaseType.not_great);
068                        operatorList.put("!<", TBaseType.not_less);   // SQL Server style not less than
069                        operatorList.put("!>", TBaseType.not_great);  // SQL Server style not greater than
070                        operatorList.put("**", TBaseType.exponentiate);
071                        operatorList.put("||", TBaseType.concatenationop);
072
073                        if (TBaseType.enterprise_edition||TBaseType.teradata_edition){
074                                inittable();
075                        }
076    }
077
078    public TLexerTeradata(){
079          super();
080          dbvendor = EDbVendor.dbvteradata;
081          sourcetokens = new TSourceTokenList();
082    }
083
084    public void setTeradataUtilityType(TeradataUtilityType teradataUtilityType) {
085        this.teradataUtilityType = teradataUtilityType;
086    }
087
088    private TeradataUtilityType teradataUtilityType = TeradataUtilityType.BTEQ;
089    
090    public TSourceTokenList sourcetokens;
091
092
093
094    int issystemvariable(String str){
095        return -1;// -1 means not a system variable
096    }
097
098public boolean canBeColumnName(int tokencode){
099    //http://blog.csdn.net/superbeck/article/details/5387476
100    boolean ret = false;
101    int modifiers = keyword_type_identifier | keyword_type_column ;
102    Integer s = keywordTypeList.get(tokencode);
103     if (s != null){
104        int modifier = s;
105        ret = (modifiers & modifier) == modifier;
106    }
107
108    return ret;
109}
110
111   public EFindSqlStateType cmdType(TSourceToken token){
112        
113        // use  isMultiLoadCmd,isFastExportCmd,isFastLoadCmd,isBteqCmd to find the command type,
114        String str = token.toString();
115        String str2 = null;
116        String str3 = null;
117        if (token.nextSolidToken() != null) {
118            str2 = token.nextSolidToken().toString();
119            if (token.nextSolidToken().nextSolidToken() != null) {
120              str3 = token.nextSolidToken().nextSolidToken().toString();
121            }
122        }
123
124    // First check the longest possible command (str + str2 + str3)
125    if (str3 != null) {
126        String combinedStr3 = str + " " + str2 + " " + str3;
127        int matchCount = 0;
128        boolean isMultiLoad3 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr3) != null;
129        boolean isFastExport3 = FastExportCmdType.searchFastExportCmd(combinedStr3) != null;
130        boolean isFastLoad3 = FastLoadCmdType.searchFastLoadCmd(combinedStr3) != null;
131        boolean isBteq3 = BteqCmdType.searchBteqCmd(combinedStr3) != null;
132
133        if (isMultiLoad3) matchCount++;
134        if (isFastExport3) matchCount++;
135        if (isFastLoad3) matchCount++;
136        if (isBteq3) matchCount++;
137
138        if (matchCount > 1) {
139            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq3) return EFindSqlStateType.stBTEQCmd;
140            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
141            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport3) return EFindSqlStateType.stFastExportCmd;
142            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
143
144            // throw new IllegalStateException("Ambiguous command '" + combinedStr3 + "' matches multiple utility types");
145            matchCount = 1;
146        }
147
148        if (matchCount == 1) {
149            if (isMultiLoad3) return EFindSqlStateType.stMultiLoadCmd;
150            if (isFastExport3) return EFindSqlStateType.stFastExportCmd;
151            if (isFastLoad3) return EFindSqlStateType.stFastLoadCmd;
152            if (isBteq3) return EFindSqlStateType.stBTEQCmd;
153        }
154    }
155
156    // Then check medium length command (str + str2)
157    if (str2 != null) {
158        String combinedStr2 = str + " " + str2;
159        int matchCount = 0;
160        boolean isMultiLoad2 = MultiLoadCmdType.searchMultiLoadCmd(combinedStr2) != null;
161        boolean isFastExport2 = FastExportCmdType.searchFastExportCmd(combinedStr2) != null;
162        boolean isFastLoad2 = FastLoadCmdType.searchFastLoadCmd(combinedStr2) != null;
163        boolean isBteq2 = BteqCmdType.searchBteqCmd(combinedStr2) != null;
164
165        if (isMultiLoad2) matchCount++;
166        if (isFastExport2) matchCount++;
167        if (isFastLoad2) matchCount++;
168        if (isBteq2) matchCount++;
169
170        if (matchCount > 1) {
171            if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq2) return EFindSqlStateType.stBTEQCmd;
172            if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
173            if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport2) return EFindSqlStateType.stFastExportCmd;
174            if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
175
176            //throw new IllegalStateException("Ambiguous command '" + combinedStr2 + "' matches multiple utility types");
177            matchCount = 1;
178        }
179        if (matchCount == 1) {
180            if (isMultiLoad2) return EFindSqlStateType.stMultiLoadCmd;
181            if (isFastExport2) return EFindSqlStateType.stFastExportCmd;
182            if (isFastLoad2) return EFindSqlStateType.stFastLoadCmd;
183            if (isBteq2) return EFindSqlStateType.stBTEQCmd;
184        }
185    }
186
187    // Finally check single word command (str)
188    boolean isMultiLoad = MultiLoadCmdType.searchMultiLoadCmd(str) != null;
189    boolean isFastExport = FastExportCmdType.searchFastExportCmd(str) != null;
190    boolean isFastLoad = FastLoadCmdType.searchFastLoadCmd(str) != null;
191    boolean isBteq = BteqCmdType.searchBteqCmd(str) != null;
192    int matchCount = 0;
193    if (isMultiLoad) matchCount++;
194    if (isFastExport) matchCount++;
195    if (isFastLoad) matchCount++;
196    if (isBteq) matchCount++;
197
198    if (matchCount > 1) {
199        if ((teradataUtilityType == TeradataUtilityType.BTEQ ) && isBteq) return EFindSqlStateType.stBTEQCmd;
200        if ((teradataUtilityType == TeradataUtilityType.MULTILOAD ) && isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
201        if ((teradataUtilityType == TeradataUtilityType.FASTEXPORT ) && isFastExport) return EFindSqlStateType.stFastExportCmd;
202        if ((teradataUtilityType == TeradataUtilityType.FASTLOAD ) && isFastLoad) return EFindSqlStateType.stFastLoadCmd;
203
204       // throw new IllegalStateException("Ambiguous command '" + str + "' matches multiple utility types");
205        matchCount = 1;
206    }
207
208    if (matchCount == 1) {
209      if (isMultiLoad) return EFindSqlStateType.stMultiLoadCmd;
210      if (isFastExport) return EFindSqlStateType.stFastExportCmd;
211      if (isFastLoad) return EFindSqlStateType.stFastLoadCmd;
212      if (isBteq) return EFindSqlStateType.stBTEQCmd;
213  }
214    return EFindSqlStateType.stnormal;
215
216  }
217
218
219
220
221 public  int iskeyword(String str){
222        int ret = -1;
223        Integer s = keywordValueList.get(str.toUpperCase(Locale.ENGLISH));
224        if( s != null){
225            ret = s;
226        }
227        return ret;// -1 means not a keyword
228     }
229         
230    public static EKeywordType getKeywordType(String keyword){
231        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
232    }            
233
234     public int getkeywordvalue(String keyword){
235        int ret = 0;
236        Integer s = keywordValueList.get(keyword.toUpperCase(Locale.ENGLISH));
237        if( s != null){
238            ret = s;
239         }
240        return ret;// 0 means not a keyword
241     }
242
243    static void yystateLookupConfigure() {
244        int yystates = yytl.length;
245        yystateTable = new int[257][yystates];
246
247        // initialize to empty
248        for(int i = 0; i < yystates; i++) {
249            for (int j = 0; j < 257; j++)
250                yystateTable[j][i] = -1;
251        }
252
253        for(int i = 0; i < yystates; i++) {
254            int low = yytl[i];
255            int high = yyth[i];
256            for (int j = low; j <= high; j++) {
257                for (char c: yyt[j].cc) {
258                    yystateTable[c][i] = j;
259                }
260            }
261        }
262    }
263        
264    int yylex(){
265          int yyn;
266           while (true) { // top level while
267              yynew();
268              while (true){  //scan
269                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
270                     yymark(yyk[yyn]);
271                 }
272
273                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
274                    yymatch(yym[yyn]);
275                 }
276
277                 if(yytl[yystate] > yyth[yystate]){
278                     break;
279                 }
280
281                 yyscan();
282//                 yyn = yytl[yystate];
283                 totablechar();
284//                 while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
285//                   yyn++;
286//                 }
287//                 if (yyn > yyth[yystate]){
288//                     break;
289//                 }
290
291                 yyn = yystateTable[yytablechar][yystate];
292                 if (yyn == -1)
293                     break;
294                                         
295                 yystate = yyt[yyn].s;
296              } //scan
297
298              while (true){ //action
299                int yyrule;
300                if ( (yyrule = yyfind()) != -1 ){
301                   yyaction(yyrule);
302                   if (yyreject){
303                       continue;
304                   }
305                }else if( (!yydefault() ) && (yywrap()) ){
306                   yyclear();
307                   returni(0);
308                }
309                break;
310              }
311
312              if (!yydone) {
313                  continue;
314              }
315              break;
316            } // top level while
317
318           return yyretval;
319        }
320
321    
322
323    static void inittable(){
324                
325                //if (yynmarks > 0) return; //init table already
326
327        String line;
328        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
329        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
330        int c=0;
331        keywordValueList.clear();
332        keywordTypeList.clear();
333        
334        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerTeradata.class.getResourceAsStream(table_file)));
335
336            try{
337                while( (line = br.readLine()) != null){
338                          if (line.trim().startsWith("yynmarks=")){
339                             String[] ss = line.split("[=;]");
340                              yynmarks=Integer.parseInt(ss[1].trim());
341                              yyk = new int[yynmarks+1];
342                          }else if (line.trim().startsWith("yynmatches=")){
343                              String[] ss = line.split("[=;]");
344                               yynmatches=Integer.parseInt(ss[1].trim());
345                               yym = new int[yynmatches+1];
346                          }else if (line.trim().startsWith("yyntrans=")){
347                              String[] ss = line.split("[=;]");
348                               yyntrans=Integer.parseInt(ss[1].trim());
349                               yytint = new int[yyntrans+1];
350                               yyt = new TYytRec[yyntrans+1];
351                          }else if (line.trim().startsWith("yynstates=")){
352                              String[] ss = line.split("[=;]");
353                               yynstates=Integer.parseInt(ss[1].trim());
354                               yykl = new int[yynstates];
355                               yykh = new int[yynstates];
356                              yyml = new int[yynstates];
357                              yymh = new int[yynstates];
358                              yytl = new int[yynstates];
359                              yyth = new int[yynstates];
360                          }else if (line.trim().startsWith("<end>")){
361                              if (inyyk){
362                                  inyyk = false;
363                                 if (yynmarks+1 != yyk_count ){
364                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
365                                 }
366                              }
367                              else if(inyym){
368                                     inyym = false;
369                                    if (yynmatches+1 != yym_count ){
370                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
371                                    }
372                              }
373                              else if(inyykl){
374                                     inyykl = false;
375                                    if (yynstates != yykl_count ){
376                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
377                                    }
378                              }
379                              else if(inyykh){
380                                     inyykh = false;
381                                    if (yynstates != yykh_count ){
382                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
383                                    }
384                              }
385                              else if(inyyml){
386                                     inyyml = false;
387                                    if (yynstates != yyml_count ){
388                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
389                                    }
390                              }
391                              else if(inyymh){
392                                     inyymh = false;
393                                    if (yynstates != yymh_count ){
394                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
395                                    }
396                              }
397                              else if(inyytl){
398                                     inyytl = false;
399                                    if (yynstates != yytl_count ){
400                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
401                                    }
402                              }
403                              else if(inyyth){
404                                     inyyth = false;
405                                    if (yynstates != yyth_count ){
406                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
407                                    }
408                              }
409                              else if(inyytint){
410                                     inyytint = false;
411                                    if (yyntrans + 1 != yytint_count ){
412                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
413                                    }
414                              }
415                              else if(inyyt){
416                                     inyyt = false;
417                                    if (yyntrans+1 != yyt_count ){
418                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
419                                    }
420                              }
421                              else if(inkeyword){
422                                     inkeyword = false;
423                              }
424                          }else if(line.trim().startsWith("yyk =")){
425                             inyyk = true; 
426                          }else if(line.trim().startsWith("yym =")){
427                             inyym = true;
428                          }else if(line.trim().startsWith("yykl =")){
429                             inyykl = true;
430                          }else if(line.trim().startsWith("yykh =")){
431                             inyykh = true;
432                          }else if(line.trim().startsWith("yyml =")){
433                             inyyml = true;
434                          }else if(line.trim().startsWith("yymh =")){
435                             inyymh = true;
436                          }else if(line.trim().startsWith("yytl =")){
437                             inyytl = true;
438                          }else if(line.trim().startsWith("yyth =")){
439                             inyyth = true;
440                          }else if(line.trim().startsWith("yytint =")){
441                             inyytint = true;
442                          }else if(line.trim().startsWith("yyt =")){
443                             inyyt = true;
444                        }else if(line.trim().startsWith("keywordsvalue =")){
445                           inkeyword = true;
446                        }else if(inyyk){
447                             String[] ss = line.split("[,]");
448                               for(int j=0;j<ss.length;j++){
449                                   // System.out.println(ss[j].trim());
450                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
451                               }
452                          }else if(inyym){
453                               String[] ss = line.split("[,]");
454                                 for(int j=0;j<ss.length;j++){
455                                     // System.out.println(ss[j].trim());
456                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
457                                 }
458                          }else if(inyykl){
459                               String[] ss = line.split("[,]");
460                                 for(int j=0;j<ss.length;j++){
461                                    //  System.out.println(ss[j].trim());
462                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
463                                 }
464                          }else if(inyykh){
465                               String[] ss = line.split("[,]");
466                                 for(int j=0;j<ss.length;j++){
467                                     // System.out.println(ss[j].trim());
468                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
469                                 }
470                          }else if(inyyml){
471                               String[] ss = line.split("[,]");
472                                 for(int j=0;j<ss.length;j++){
473                                     // System.out.println(ss[j].trim());
474                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
475                                 }
476                          }else if(inyymh){
477                               String[] ss = line.split("[,]");
478                                 for(int j=0;j<ss.length;j++){
479                                     // System.out.println(ss[j].trim());
480                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
481                                 }
482                          }else if(inyytl){
483                               String[] ss = line.split("[,]");
484                                 for(int j=0;j<ss.length;j++){
485                                     // System.out.println(ss[j].trim());
486                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
487                                 }
488                          }else if(inyyth){
489                               String[] ss = line.split("[,]");
490                                 for(int j=0;j<ss.length;j++){
491                                     // System.out.println(ss[j].trim());
492                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
493                                 }
494                          }else if(inyytint){
495                               String[] ss = line.split("[,]");
496                                 for(int j=0;j<ss.length;j++){
497                                     // System.out.println(ss[j].trim());
498                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
499                                 }
500                          }else if(inyyt){
501                                //System.out.println(line.trim());
502
503                              c = 0;
504                              String[] st = line.trim().split(",,");
505                              char[] tmp = new char[st.length];
506                              for(int i=0;i<st.length;i++){
507
508                                  if(st[i].startsWith("\'")) {
509                                      if(st[i].length() == 3){  // 'a'
510                                          tmp[c++] = st[i].charAt(1);
511                                      }else if(st[i].length() == 4) { // '\\'
512                                          tmp[c++] = st[i].charAt(2);
513                                      }else{
514                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
515                                      }
516                                  }else{
517                                      try{
518                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
519                                          } catch (NumberFormatException nfe) {
520                                             System.out.println("NumberFormatException: " + nfe.getMessage());
521                                          }
522                                  }
523                              } //while hasmoreTokens
524
525                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
526                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
527                              yyt_count++;
528
529                          }else if(inkeyword){
530                              String[] ss =line.split("[=]");
531
532                              int val1 = -1;
533                              int val2 = -1;
534                              try {
535                                  val1 = Integer.parseInt(ss[1]);
536                                  val2 = Integer.parseInt(ss[2]);
537                              }
538                              catch (NumberFormatException nfe) {
539                                  System.out.println("NumberFormatException: " + nfe.getMessage());
540                              }
541                              keywordValueList.put(ss[0].toUpperCase(),val1);
542                              keywordTypeList.put(val1,val2);
543                             }
544                }
545                }catch(IOException e){
546                  System.out.println(e.toString());
547                }
548                                
549                                yystateLookupConfigure();
550
551    }
552
553
554
555    void yyaction(int yyruleno){
556
557
558
559      int ic;
560      char[] tmparray = {'=','+','-','*','/','>','<'};
561
562      yylvalstr = getyytext();
563  /* actions: */
564  switch(yyruleno){
565  case 1:
566                
567        {
568              addlit(yylvalstr,yytextlen);
569              if (xcdepth <= 0)
570                 {
571                  start(init);
572                  yylvalstr = litbufdup();
573                  returni(cmtslashstar);
574                 }
575              else
576                 xcdepth--;
577
578            break;
579        }
580
581
582  case 2:
583                
584          {
585             // xcdepth++;
586              yyless(1);
587              addlit(yylvalstr,1);
588              break;
589           }
590
591  case 3:
592                        
593          {
594
595              if (getyysstate() == xq)
596              {
597                  nchars = yytextlen;
598                  addlit(yylvalstr, yytextlen-1);
599                  yyless(nchars-1);
600                  return;//exit;
601              }
602
603              xcdepth = 0;
604              start(xc);
605              startlit();
606              yyless(2);
607              addlit(yylvalstr,yytextlen);
608
609          break;
610          }
611
612  case 4:
613                
614          {
615            addlit(yylvalstr,yytextlen);
616            break;
617          }
618
619  case 5:
620                
621          {
622              addlitchar(yylvalstr.charAt(0));
623              break;
624          }
625
626                  
627  case 6:
628                       
629        {
630                                        start(init);
631                                        addlit(yylvalstr, yytextlen);
632                                        yylvalstr = litbufdup();
633                                        returni(ident);
634                break;
635        }
636
637  case 7:
638                        
639        {
640          if ((getyysstate() == xd)||(getyysstate() == xq))
641          {
642              addlit(yylvalstr, 1);
643              yyless(1);
644              return;//exit;
645          }     
646                                        //token_start := yylvalStr;
647                                        start(xbacktick);
648                                        startlit();
649                                        addlit(yylvalstr, yytextlen);
650                break;
651        }
652
653
654  case 8:
655                       
656        {
657                addlit(yylvalstr, yytextlen);
658                break;
659  }
660
661  case 9:
662                       
663        {
664    addlit(yylvalstr, yytextlen);
665                break;
666   }
667                  
668
669  case 10:
670                  
671          {
672            returni(ident);
673            break;
674          }
675          
676  case 11:
677                           
678                        {
679                                if ((getyysstate() == xq)){
680                      addlit(yylvalstr, 1);
681                      yyless(1);
682                      return;
683                                }else{
684                                        returni(sconst);
685                                        break;
686                                }
687                        }
688                        
689  case 12:
690                           
691                        {
692                                if ((getyysstate() == xq)){
693                      addlit(yylvalstr, 1);
694                      yyless(1);
695                      return;
696                                }else{
697                                        returni(TBaseType.teradata_unicode_sconst);
698                                        break;
699                                }
700                        }
701                        
702                        
703  case 13:
704                
705                        {
706                                if ((getyysstate() == xq)){
707                      addlit(yylvalstr, 1);
708                      yyless(1);
709                      return;
710                                }else{
711                                        returni(sconst);
712                                        break;
713                                }
714                        }
715                        
716  case 14:
717                                        
718                        {
719                        returni(sconst);
720                        break;
721                        }                       
722                        
723  case 15:
724                
725                        {
726                        returni(not_equal);
727                        break;
728                        }                       
729
730  case 16:
731                
732          {
733              start(init);
734              addlit(yylvalstr, yytextlen);
735              yylvalstr = litbufdup();
736              returni(sconst);
737              break;
738          }
739
740  case 17:
741                           
742          {
743              start(init);
744              addlit(yylvalstr, yytextlen);
745              yylvalstr = litbufdup();
746              returni(sconst);
747              break;
748          }
749
750  case 18:
751                
752          {
753            addlit(yylvalstr, yytextlen);
754            break;
755          }
756
757
758  case 19:
759                        
760          {
761              if (yysstate == xd)
762                {
763                  addlit(yylvalstr, yytextlen);
764                }
765              else
766                {
767                  start(xq);
768                  startlit();
769                  addlit(yylvalstr, yytextlen);
770                  dummych1 = get_char();
771                  if (dummych1 == '\\') // recognize string like '\'
772                    {
773                      dummych2 = get_char();
774                      if (dummych2 == '\'')
775                      {
776                          start(init);
777                          addlit("\\", 1);
778                          addlit("\'", 1);
779                          yylvalstr = litbufdup();
780                          returni(sconst);
781                      }
782                      else
783                       {
784                          unget_char(dummych2);
785                          unget_char(dummych1);
786                        }
787                    }
788                  else
789                    unget_char(dummych1);
790                }
791
792              break;
793          }
794
795  case 20:
796                
797          {
798              addlit(yylvalstr, yytextlen);
799              break;
800          }
801
802  case 21:
803                
804          {
805            dummych1 = get_char();
806            unget_char(dummych1);
807            if (dummych1 == (char)10)
808              {
809                dummych1 = get_char();
810                addlit(yylvalstr+dummych1, yytextlen+1);
811              } else if (dummych1 == '\\')
812                  {
813                      dummych1 = get_char();
814                      dummych2 = get_char();
815                      if (dummych2 == '\'')
816                        {
817                          unget_char(dummych2);
818                          addlit(yylvalstr+dummych1, yytextlen+1);
819                        }
820                      else
821                        addlit(yylvalstr+dummych1+dummych2, yytextlen+2);
822                  }
823                else
824                  addlit(yylvalstr, yytextlen);
825
826          break;
827         }
828        
829  case 22:
830                 
831          {
832              addlit(yylvalstr, yytextlen);
833              break;
834          }
835
836
837
838  case 23:
839                
840          {
841              start(init);
842              addlit(yylvalstr, yytextlen);
843              if (literallen == 0)   returni (error);
844              if (literallen >= namedatalen)
845              {
846                 setlengthofliteralbuf(namedatalen);
847                 literallen = namedatalen;
848              }
849              yylvalstr = litbufdup();
850              returni (ident);
851
852              break;
853          }
854
855  case 24:
856               
857          {
858          addlit(yylvalstr, yytextlen);
859          break;
860          }
861
862  case 25:
863                        
864          {
865              start(xd);
866              startlit();
867              addlit(yylvalstr, yytextlen);
868              break;
869          }
870        
871  case 26:
872                
873        {
874                
875                dummych1 = get_char();
876                unget_char(dummych1);
877                if (dummych1 == (char)10)
878                  {
879                dummych1 = get_char();
880                addlit(yylvalstr+dummych1, yytextlen+1);
881                  } else
882                    addlit(yylvalstr, yytextlen);
883                
884                break;
885        }
886
887
888  case 27:
889                
890          {
891            returni(lexnewline);
892            break;
893          }
894
895  case 28:
896        
897          {
898              returni(lexspace);
899              break;
900          }
901
902  case 29:
903                
904          {
905          if ((getyysstate() == xq)
906              || (getyysstate() == xd)
907              || (getyysstate() == xc)
908              )
909          {
910              addlit(yylvalstr, 1);
911              yyless(1);
912              return;//exit;
913          }
914
915          returni(cmtdoublehyphen);
916          break;
917          }
918
919  case 30:
920        
921          {
922               returnc(yylvalstr.charAt(0));
923               break;
924          }
925
926  case 31:
927                       
928          {
929            returni(cmpop);
930            break;
931          }
932        
933  case 32:
934                  
935      {
936        returni(concatenationop);
937        break;
938      }
939          
940  case 33:
941           
942          {
943
944          if (getyysstate() == xc)
945             {
946              slashstar = yylvalstr.indexOf("*/");
947                if (slashstar >= 0)
948                  {
949                      addlit(yylvalstr,slashstar+2);
950                      yylvalstr = litbufdup();
951                      yyless(slashstar+2);
952
953                                                                                        if (xcdepth <= 0){
954                                                                                                start(init);
955                                                                                                returni(cmtslashstar);
956                                                                                        }else{
957                                                                                                xcdepth--;
958                                                                                        }
959                  }
960                else
961                  {
962                      addlit(yylvalstr,1);
963                      yyless(1);
964                  }
965             }
966          else
967            {
968              nchars = yytextlen;
969              slashstar = yylvalstr.indexOf("/*");
970              dashdash = yylvalstr.indexOf("--");
971              if ((slashstar >= 0) && (dashdash >= 0))
972                {
973                  //if both appear, take the first one
974                   if (slashstar > dashdash)
975                    {slashstar = dashdash;}
976                }
977              else
978                {
979                  //   if slashstar=0 then slashstar := dashdash;
980                  // add (getyysstate <> xc) to avoid something like this */--,here */ should be handled instead of --
981                  if ((slashstar > 0) && (getyysstate() != xc)) {
982                    nchars = slashstar;
983                    }
984                }
985
986              // Search for known operators from longest to shortest
987              // This handles cases like <=-.01 where <= should be returned, not <=-
988              boolean searchOp = false;
989              while (nchars > 0){
990                  String opStr = yylvalstr.substring(0,nchars);
991                  Integer opCode = operatorList.get(opStr);
992                  if (opCode != null){
993                      if (nchars < yytextlen){
994                          yyless(nchars);
995                          yylvalstr = yylvalstr.substring(0,nchars);
996                      }
997                      searchOp = true;
998                      returni(opCode);
999                      break;
1000                  }
1001                  nchars--;
1002              }
1003
1004              if (searchOp) break;
1005
1006              nchars = yytextlen; // reset nchars
1007
1008              while ((nchars > 1)
1009                      && ( (yylvalstr.charAt(nchars-1) == '+' )
1010                          || (yylvalstr.charAt(nchars-1) =='-'))
1011                      && (getyysstate() != xc))
1012                {
1013                  for (ic = nchars - 1; ic>=1; ic--)
1014                  {
1015                    if (isopchar(yylvalstr.charAt(ic-1)))  break;
1016                  }
1017                  if (ic >= 1) break;
1018                  nchars--;
1019                }
1020
1021              if (nchars < yytextlen)
1022                {
1023                  //Strip the unwanted chars from the token
1024                  yyless(nchars);
1025                  yylvalstr = yylvalstr.substring(0,nchars);
1026                }
1027
1028                  ///*
1029                  // * If what we have left is only one char, and it's
1030                  // * one of the characters matching "self", then
1031                  // * return it as a character token the same way
1032                  // * that the "self" rule would have.
1033                  // * make sure @ return as self char, by james wang
1034                  // */
1035                  if ((nchars == 1) && (isselfchar(yylvalstr.charAt(0)) || (yylvalstr.charAt(0) == '@')))
1036                    {
1037                      returnc(yylvalstr.charAt(0));
1038                    }
1039                  else if ( (nchars >= 2) && (yylvalstr.charAt(0) == '.') )
1040                    {
1041                        yyless(1);
1042                        returnc(yylvalstr.charAt(0));
1043                    }
1044                  else if (
1045                              (nchars >= 2)
1046                                  &&(
1047                                       charinarray(yylvalstr.charAt(0), tmparray)
1048                                       && (yylvalstr.charAt(1) == ':')
1049                                       )
1050                           )
1051                      {
1052                        yyless(nchars-1);
1053                         yylvalstr = yylvalstr.substring(0,nchars-1);
1054                        if (nchars == 2)
1055                          returnc(yylvalstr.charAt(0));
1056                        else
1057                          returni(op);
1058                      }
1059                    else if (
1060                                (nchars >= 2)
1061                                    && (
1062                                          charinarray(yylvalstr.charAt(nchars-1-1),tmparray)
1063                                          && (yylvalstr.charAt(nchars-1) == '.')
1064                                         )
1065                               )
1066                        {
1067                          yyless(nchars-1);
1068                           yylvalstr = yylvalstr.substring(0,nchars-1);
1069                          if (nchars == 2)
1070                            returnc(yylvalstr.charAt(0));
1071                          else
1072                            returni(op);
1073                        }
1074                    else if ( (nchars == 2) && ((yylvalstr.charAt(nchars-1) == '$')||(yylvalstr.charAt(nchars-1) == '?')) )
1075                      {
1076                        yyless(nchars-1);
1077                        yylvalstr = yylvalstr.substring(0,nchars-1);
1078                        returnc(yylvalstr.charAt(0));
1079                      }
1080                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == '@') )
1081                      {
1082                        if (nchars == 2)
1083                          { // such as >@
1084                              yyless(nchars-1);
1085                            yylvalstr = yylvalstr.substring(0,nchars-1);
1086                             returnc(yylvalstr.charAt(0));
1087                          }
1088                        else
1089                          {
1090                            if (yylvalstr.charAt(nchars-1-1) == '@')
1091                              { //such as >@@,>=@@
1092                                  yyless(nchars-2);
1093                                yylvalstr = yylvalstr.substring(0,nchars-2);
1094                                if (nchars == 3)
1095                                    returnc(yylvalstr.charAt(0));
1096                                else
1097                                    returni(cmpop);
1098                              }
1099                            else
1100                              { // >=@
1101                                  yyless(nchars-1);
1102                                yylvalstr = yylvalstr.substring(0,nchars-1);
1103                                  returni(cmpop);
1104                              }
1105                          }
1106                         }
1107                    else if ( (nchars >= 2) && (yylvalstr.charAt(nchars-1) == ':') )
1108                      {
1109                        if (nchars == 2)
1110                          { // such as >:
1111                            yyless(nchars-1);
1112                            yylvalstr = yylvalstr.substring(0,nchars-1);
1113                            returnc(yylvalstr.charAt(0));
1114                          }
1115                        else
1116                          {
1117                          // >=:
1118                                  yyless(nchars-1);
1119                                yylvalstr = yylvalstr.substring(0,nchars-1);
1120                                                        if (yylvalstr.equalsIgnoreCase("||")){
1121                                  returni(concatenationop);
1122                              }else                                                             
1123                                  returni(cmpop);
1124                         }
1125                       
1126                         }
1127                                         else if ( (nchars >= 2) && (yylvalstr.charAt(0)== '|') && (yylvalstr.charAt(1) == '|'))
1128                        { //||--sss ,just get || only
1129                              yyless(2);
1130                              yylvalstr = yylvalstr.substring(0,2);
1131                              returni(concatenationop); //return as concatenationop, this is used in sybase only
1132                        }
1133                      else if ((nchars == 2) && ((yylvalstr.charAt(0) =='.') && ( (yylvalstr.charAt(1)=='*') || (yylvalstr.charAt(1) =='#')) ))
1134                          {
1135                            yyless(1);
1136                            returnc(yylvalstr.charAt(0));
1137                          }
1138                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '#')) ))
1139                          {
1140                            yyless(1);
1141                            returnc(yylvalstr.charAt(0));
1142                          }
1143                      else if ((nchars == 2) && ((yylvalstr.charAt(0) == '=') && ( (yylvalstr.charAt(1) == '?')) ))
1144                          {
1145                            yyless(1);
1146                            returnc(yylvalstr.charAt(0));
1147                          }
1148                                        else if ( (nchars >= 3) && (yylvalstr.charAt(nchars-1) == '$' ) && (yylvalstr.charAt(nchars-2) == '$' ))
1149                                        { // =$$abc
1150                                            if (nchars == 3){
1151                                                        yyless(1);
1152                                                        returnc(yylvalstr.charAt(0));
1153                                                }else{
1154                                                        yyless(nchars-2);
1155                                                        yylvalstr = yylvalstr.substring(0,nchars-2);
1156                                                        returni(cmpop);
1157                                                }
1158                                        }
1159                                                  else if (((nchars > 2) && (yylvalstr.charAt(0) == '*'))
1160                              && (yylvalstr.charAt(1) == '/')
1161                              && (getyysstate() == xc)
1162                              )
1163                           { //in comment, and find */ , then it must the end of comment
1164                             yyless(2);
1165                                addlit(yylvalstr,yytextlen);
1166                             if (xcdepth <= 0)
1167                               {
1168                                 start(init);
1169                                 yylvalstr = litbufdup();
1170                                 returni(cmtslashstar);
1171                               }
1172                             else
1173                               xcdepth--;
1174                           }
1175                      else
1176                        returni(op);
1177               }
1178
1179          break;
1180          }
1181
1182  case 34:
1183                        
1184          {
1185               returni(iconst);
1186               break;
1187          }
1188
1189  case 35:
1190                        
1191          {
1192            ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
1193            nchars = yytextlen;
1194            if (yylvalstr.charAt(nchars-1) == '.')
1195              {
1196                dummych1 = get_char();
1197                unget_char(dummych1);
1198                if (dummych1 == '.')
1199                  {
1200                      yyless(nchars-1);
1201                    yylvalstr = yylvalstr.substring(0,nchars - 1);
1202                    returni (iconst);
1203                    return;//exit;
1204                  }
1205              }
1206            returni (fconst);
1207           break;
1208          }
1209
1210  case 36:
1211                        
1212          {
1213           returni (fconst);
1214           break;
1215          }
1216
1217  case 37:
1218              
1219          {
1220            int rw;
1221            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
1222            else
1223               returni(ident);
1224            break;
1225          }
1226
1227
1228  case 38:
1229           
1230        {
1231                if (yylvalstr.charAt(0) == '$'){
1232                        returni(ident);
1233                }else
1234                        returni(variable);
1235
1236                break;
1237        }
1238                  
1239  case 39:
1240                  
1241          {
1242            if ((yylvalstr.charAt(0) == '$')&&(yylvalstr.charAt(1) == '_')){
1243              returni(ident);
1244            }else
1245              returni(variable);
1246
1247            break;
1248          }
1249 
1250  case 40:
1251           
1252          {
1253            returni(mslabel);
1254            break;
1255          }
1256
1257                  
1258  case 41:
1259               
1260          {
1261            returni(lexspace);
1262            break;
1263          }               
1264                  
1265  case 42:
1266                    
1267          {
1268            returni(UNICODE_ENCODE_ID);
1269            break;
1270          }               
1271                  
1272    
1273  case 43:
1274                        
1275          {
1276            returni( error);
1277            break;
1278          }
1279    default:{
1280     System.out.println("fatal error in yyaction");
1281    }
1282   }//switch
1283}/*yyaction*/;
1284
1285
1286
1287        }