001// lexical analyzer  for GSQLParser component java version
002
003/****************************************************}
004{   Lexical analizer for GSQLParser component java version    }
005{   Copyright (c) 2004-2016 by Gudu software              }
006{****************************************************/
007
008package gudusoft.gsqlparser;
009
010import gudusoft.gsqlparser.nodes.TTypeName;
011
012import java.util.HashMap;
013import java.io.InputStreamReader;
014
015import java.util.Locale;
016import java.io.BufferedReader;
017import java.io.IOException;
018
019
020
021public class TLexerDax extends TCustomLexer{
022
023    static int yynmarks = 0  ;
024    static int yynmatches ;
025    static int yyntrans   ;
026    static int yynstates  ;
027    static int[]  yyk,yym ; // 1 based
028    static int[]  yytint;  // 1 based
029    static TYytRec[] yyt ;  // 1 based
030    static int[]  yykl,yykh,yyml,yymh,yytl,yyth ; // 0 based
031    private  static  String[] keywordlist;
032    static String   table_file;
033        static HashMap keywordValueList;
034        static HashMap keywordTypeList;
035
036    static {
037        keywordValueList = new HashMap();
038        keywordTypeList = new HashMap();
039        table_file = "/gudusoft/gsqlparser/parser/dax/dax_lex_table.txt";
040                if (TBaseType.enterprise_edition||TBaseType.dax_edition){
041                inittable();
042            }
043    }
044
045    public TLexerDax(){
046          super();
047          dbvendor = EDbVendor.dbvdax;
048          sourcetokens = new TSourceTokenList();
049    }
050
051
052    public TSourceTokenList sourcetokens;
053
054
055public boolean canBeColumnName(int tokencode){
056    //http://blog.csdn.net/superbeck/article/details/5387476
057    boolean ret = false;
058    int modifiers = keyword_type_identifier | keyword_type_column ;
059    String s = (String)keywordTypeList.get(Integer.toString(tokencode));
060    if (s != null){
061        int modifier = Integer.parseInt(s);
062        ret = (modifiers & modifier) == modifier;
063    }
064
065    return ret;
066}
067
068public  int iskeyword(String str){
069       int ret = -1;
070       String s = (String) keywordValueList.get(str.toUpperCase(new Locale("Locale.US")));
071       if( s != null){
072           try {
073                  ret = Integer.parseInt(s);
074                 // System.out.println(str+" is keyword, value: "+ret);
075                 } catch (NumberFormatException nfe) {
076                    return -1;
077                 }
078       }
079       return ret;// -1 means not a keyword
080    }
081        
082    public static EKeywordType getKeywordType(String keyword){
083        return TCustomLexer.getKeywordType(keyword,keywordValueList,keywordTypeList);
084    }           
085
086    public int getkeywordvalue(String keyword){
087        try{
088                if (keywordValueList.get(keyword.toUpperCase()) == null) return 0;
089         return Integer.parseInt((String) keywordValueList.get(keyword.toUpperCase()));
090        }catch(NumberFormatException e){
091            return 0;
092        }
093    }
094
095
096    int yylex(){
097          int yyn;
098           while (true) { // top level while
099              yynew();
100              while (true){  //scan
101                 for(yyn = yykl[yystate]; yyn <= yykh[yystate]; yyn++){
102                     yymark(yyk[yyn]);
103                 }
104
105                 for(yyn=yymh[yystate]; yyn>= yyml[yystate]; yyn--){
106                    yymatch(yym[yyn]);
107                 }
108
109                 if(yytl[yystate] > yyth[yystate]){
110                     break;
111                 }
112
113                 yyscan();
114                 yyn = yytl[yystate];
115                 totablechar();
116                 while( (yyn <= yyth[yystate]) && (!(charinarray(yytablechar,yyt[yyn].cc))) ){
117                   yyn++;
118                 }
119                 if (yyn > yyth[yystate]){
120                     break;
121                 }
122                 yystate = yyt[yyn].s;
123              } //scan
124
125              while (true){ //action
126                int yyrule;
127                if ( (yyrule = yyfind()) != -1 ){
128                   yyaction(yyrule);
129                   if (yyreject){
130                       continue;
131                   }
132                }else if( (!yydefault() ) && (yywrap()) ){
133                   yyclear();
134                   returni(0);
135                }
136                break;
137              }
138
139              if (!yydone) {
140                  continue;
141              }
142              break;
143            } // top level while
144
145           return yyretval;
146        }
147
148    static void inittable(){
149                
150                //if (yynmarks > 0) return; //init table already
151
152        String line;
153        boolean inyyk=false,inyym=false,inyykl=false,inyykh=false,inyyml=false,inyymh=false,inyytl=false,inyyth=false,inyytint=false,inyyt=false,inkeyword=false;
154        int yyk_count=0,yym_count=0,yykl_count=0,yykh_count=0,yyml_count=0,yymh_count=0,yytl_count=0,yyth_count=0,yytint_count=0,yyt_count=0;
155        int c=0;
156        keywordValueList.clear();
157        keywordTypeList.clear();
158        
159        BufferedReader br = new BufferedReader(new InputStreamReader(TLexerDax.class.getResourceAsStream(table_file)));
160
161            try{
162                while( (line = br.readLine()) != null){
163                          if (line.trim().startsWith("yynmarks=")){
164                             String[] ss = line.split("[=;]");
165                              yynmarks=Integer.parseInt(ss[1].trim());
166                              yyk = new int[yynmarks+1];
167                          }else if (line.trim().startsWith("yynmatches=")){
168                              String[] ss = line.split("[=;]");
169                               yynmatches=Integer.parseInt(ss[1].trim());
170                               yym = new int[yynmatches+1];
171                          }else if (line.trim().startsWith("yyntrans=")){
172                              String[] ss = line.split("[=;]");
173                               yyntrans=Integer.parseInt(ss[1].trim());
174                               yytint = new int[yyntrans+1];
175                               yyt = new TYytRec[yyntrans+1];
176                          }else if (line.trim().startsWith("yynstates=")){
177                              String[] ss = line.split("[=;]");
178                               yynstates=Integer.parseInt(ss[1].trim());
179                               yykl = new int[yynstates];
180                               yykh = new int[yynstates];
181                              yyml = new int[yynstates];
182                              yymh = new int[yynstates];
183                              yytl = new int[yynstates];
184                              yyth = new int[yynstates];
185                          }else if (line.trim().startsWith("<end>")){
186                              if (inyyk){
187                                  inyyk = false;
188                                 if (yynmarks+1 != yyk_count ){
189                                    System.out.println("required1:"+(yynmarks)+" actually:"+(yyk_count-1));
190                                 }
191                              }
192                              else if(inyym){
193                                     inyym = false;
194                                    if (yynmatches+1 != yym_count ){
195                                       System.out.println("required2:"+(yynmatches)+" actually:"+(yym_count-1));
196                                    }
197                              }
198                              else if(inyykl){
199                                     inyykl = false;
200                                    if (yynstates != yykl_count ){
201                                       System.out.println("required3:"+(yynstates)+" actually:"+(yykl_count));
202                                    }
203                              }
204                              else if(inyykh){
205                                     inyykh = false;
206                                    if (yynstates != yykh_count ){
207                                       System.out.println("required4:"+(yynstates)+" actually:"+(yykh_count));
208                                    }
209                              }
210                              else if(inyyml){
211                                     inyyml = false;
212                                    if (yynstates != yyml_count ){
213                                       System.out.println("required5:"+(yynstates)+" actually:"+(yyml_count));
214                                    }
215                              }
216                              else if(inyymh){
217                                     inyymh = false;
218                                    if (yynstates != yymh_count ){
219                                       System.out.println("required:"+(yynstates)+" actually:"+(yymh_count));
220                                    }
221                              }
222                              else if(inyytl){
223                                     inyytl = false;
224                                    if (yynstates != yytl_count ){
225                                       System.out.println("required6:"+(yynstates)+" actually:"+(yytl_count));
226                                    }
227                              }
228                              else if(inyyth){
229                                     inyyth = false;
230                                    if (yynstates != yyth_count ){
231                                       System.out.println("required7:"+(yynstates)+" actually:"+(yyth_count));
232                                    }
233                              }
234                              else if(inyytint){
235                                     inyytint = false;
236                                    if (yyntrans + 1 != yytint_count ){
237                                       System.out.println("required8:"+(yyntrans)+" actually:"+(yytint_count-1));
238                                    }
239                              }
240                              else if(inyyt){
241                                     inyyt = false;
242                                    if (yyntrans+1 != yyt_count ){
243                                       System.out.println("required9:"+(yyntrans)+" actually:"+(yyt_count-1));
244                                    }
245                              }
246                              else if(inkeyword){
247                                     inkeyword = false;
248                              }
249                          }else if(line.trim().startsWith("yyk =")){
250                             inyyk = true; 
251                          }else if(line.trim().startsWith("yym =")){
252                             inyym = true;
253                          }else if(line.trim().startsWith("yykl =")){
254                             inyykl = true;
255                          }else if(line.trim().startsWith("yykh =")){
256                             inyykh = true;
257                          }else if(line.trim().startsWith("yyml =")){
258                             inyyml = true;
259                          }else if(line.trim().startsWith("yymh =")){
260                             inyymh = true;
261                          }else if(line.trim().startsWith("yytl =")){
262                             inyytl = true;
263                          }else if(line.trim().startsWith("yyth =")){
264                             inyyth = true;
265                          }else if(line.trim().startsWith("yytint =")){
266                             inyytint = true;
267                          }else if(line.trim().startsWith("yyt =")){
268                             inyyt = true;
269                        }else if(line.trim().startsWith("keywordsvalue =")){
270                           inkeyword = true;
271                        }else if(inyyk){
272                             String[] ss = line.split("[,]");
273                               for(int j=0;j<ss.length;j++){
274                                   // System.out.println(ss[j].trim());
275                                 yyk[yyk_count++] = Integer.parseInt(ss[j].trim());
276                               }
277                          }else if(inyym){
278                               String[] ss = line.split("[,]");
279                                 for(int j=0;j<ss.length;j++){
280                                     // System.out.println(ss[j].trim());
281                                   yym[yym_count++] = Integer.parseInt(ss[j].trim());
282                                 }
283                          }else if(inyykl){
284                               String[] ss = line.split("[,]");
285                                 for(int j=0;j<ss.length;j++){
286                                    //  System.out.println(ss[j].trim());
287                                   yykl[yykl_count++] = Integer.parseInt(ss[j].trim());
288                                 }
289                          }else if(inyykh){
290                               String[] ss = line.split("[,]");
291                                 for(int j=0;j<ss.length;j++){
292                                     // System.out.println(ss[j].trim());
293                                   yykh[yykh_count++] = Integer.parseInt(ss[j].trim());
294                                 }
295                          }else if(inyyml){
296                               String[] ss = line.split("[,]");
297                                 for(int j=0;j<ss.length;j++){
298                                     // System.out.println(ss[j].trim());
299                                   yyml[yyml_count++] = Integer.parseInt(ss[j].trim());
300                                 }
301                          }else if(inyymh){
302                               String[] ss = line.split("[,]");
303                                 for(int j=0;j<ss.length;j++){
304                                     // System.out.println(ss[j].trim());
305                                   yymh[yymh_count++] = Integer.parseInt(ss[j].trim());
306                                 }
307                          }else if(inyytl){
308                               String[] ss = line.split("[,]");
309                                 for(int j=0;j<ss.length;j++){
310                                     // System.out.println(ss[j].trim());
311                                   yytl[yytl_count++] = Integer.parseInt(ss[j].trim());
312                                 }
313                          }else if(inyyth){
314                               String[] ss = line.split("[,]");
315                                 for(int j=0;j<ss.length;j++){
316                                     // System.out.println(ss[j].trim());
317                                   yyth[yyth_count++] = Integer.parseInt(ss[j].trim());
318                                 }
319                          }else if(inyytint){
320                               String[] ss = line.split("[,]");
321                                 for(int j=0;j<ss.length;j++){
322                                     // System.out.println(ss[j].trim());
323                                   yytint[yytint_count++] = Integer.parseInt(ss[j].trim());
324                                 }
325                          }else if(inyyt){
326                                //System.out.println(line.trim());
327
328                              c = 0;
329                              String[] st = line.trim().split(",,");
330                              char[] tmp = new char[st.length];
331                              for(int i=0;i<st.length;i++){
332
333                                  if(st[i].startsWith("\'")) {
334                                      if(st[i].length() == 3){  // 'a'
335                                          tmp[c++] = st[i].charAt(1);
336                                      }else if(st[i].length() == 4) { // '\\'
337                                          tmp[c++] = st[i].charAt(2);
338                                      }else{
339                                         System.out.println(" read yytstr error, error string is "+st[i]+ "line: "+ yyt_count);
340                                      }
341                                  }else{
342                                      try{
343                                           tmp[c++] = (char)Integer.parseInt(st[i]);   // char in number like 32 that represent space
344                                          } catch (NumberFormatException nfe) {
345                                             System.out.println("NumberFormatException: " + nfe.getMessage());
346                                          }
347                                  }
348                              } //while hasmoreTokens
349
350                          //yyt[lineno] = new YYTrec(tmp,yytint[lineno]);
351                              yyt[yyt_count] = new TYytRec(tmp,yytint[yyt_count]);
352                              yyt_count++;
353
354                          }else if(inkeyword){
355                              String[] ss =line.split("[=]");
356
357                              keywordValueList.put(ss[0].toUpperCase(),ss[1]);
358                              keywordTypeList.put(ss[1],ss[2]);
359                             }
360                }
361                }catch(IOException e){
362                  System.out.println(e.toString());
363                }
364
365    }
366
367
368
369    void yyaction(int yyruleno){
370        
371        
372
373      int ic;
374      char[] tmparray = {'=','+','-','*','/','>','<'};
375
376      yylvalstr = getyytext();
377  /* actions: */
378  switch(yyruleno){
379  case 1:
380                
381        {
382           addlit(yylvalstr,yytextlen);
383           if (xcdepth <= 0)
384              {
385               start(init);
386               yylvalstr = litbufdup();
387               returni(cmtslashstar);
388              }
389           else
390              xcdepth--;
391
392         break;
393        }
394
395
396  case 2:
397                
398          {
399              xcdepth++;
400              yyless(2);
401              addlit(yylvalstr,yytextlen);
402              break;
403           }
404
405  case 3:
406                        
407          {
408
409              if (getyysstate() == xq)
410              {
411                  nchars = yytextlen;
412                  addlit(yylvalstr, yytextlen-1);
413                  yyless(nchars-1);
414                  return;//exit;
415              }
416
417              xcdepth = 0;
418              start(xc);
419              startlit();
420              yyless(2);
421              addlit(yylvalstr,yytextlen);
422
423          break;
424          }
425
426  case 4:
427                
428          {
429            addlit(yylvalstr,yytextlen);
430            break;
431          }
432
433  case 5:
434                
435          {
436              addlitchar(yylvalstr.charAt(0));
437              break;
438          }
439
440  case 6:
441                
442          {
443              start(init);
444              addlit(yylvalstr, yytextlen);
445              yylvalstr = litbufdup();
446              returni(ident);
447              break;
448          }
449
450  case 7:
451                
452          {
453            addlit(yylvalstr, yytextlen);
454            break;
455          }
456
457  case 8:
458            
459          {
460              if (getyysstate() == xq)
461              {
462                  nchars = yytextlen;
463                  addlit(yylvalstr, yytextlen-1);
464                  yyless(nchars-1);
465                  return;//exit;
466              }
467
468              start(xq);
469              startlit();
470              addlit(yylvalstr,yytextlen);
471              break;
472          }
473
474  case 9:
475                        
476          {
477              if (yysstate == xd)
478                {
479                  addlit(yylvalstr, yytextlen);
480                }
481              else
482                {
483                  start(xq);
484                  startlit();
485                  addlit(yylvalstr, yytextlen);
486                }
487
488              break;
489          }
490
491  case 10:
492                
493          {
494              addlit(yylvalstr, yytextlen);
495              break;
496          }
497
498  case 11:
499                
500          {
501            dummych1 = get_char();
502            unget_char(dummych1);
503            if (dummych1 == (char)10)
504              {
505                dummych1 = get_char();
506                addlit(yylvalstr+dummych1, yytextlen+1);
507              } 
508                else
509                  addlit(yylvalstr, yytextlen);
510
511          break;
512         }
513        
514
515  case 12:
516                
517          {
518              start(init);
519              addlit(yylvalstr, yytextlen);
520              if (literallen == 0)   returni (error);
521              if (literallen >= namedatalen)
522              {
523                 setlengthofliteralbuf(namedatalen);
524                 literallen = namedatalen;
525              }
526              yylvalstr = litbufdup();
527              returni (sconst);
528
529              break;
530          }
531
532  case 13:
533               
534          {
535          addlit(yylvalstr, yytextlen);
536          break;
537          }
538
539  case 14:
540                        
541          {
542              start(xd);
543              startlit();
544              addlit(yylvalstr, yytextlen);
545              break;
546          }
547        
548  case 15:
549                
550          {
551            dummych1 = get_char();
552            unget_char(dummych1);
553            if (dummych1 == (char)10)
554              {
555                    dummych1 = get_char();
556                    addlit(yylvalstr+dummych1, yytextlen+1);
557              } else
558                addlit(yylvalstr, yytextlen);
559
560          break;
561          }
562
563  case 16:
564                              
565          {
566              dummych1 = get_char();
567              unget_char(dummych1);
568              if (dummych1 == ']')
569                  {
570                      dummych1 = get_char();
571                      addlit(yylvalstr+dummych1, yytextlen+1);
572                  }
573              else
574                  {
575                                                                                        
576                      start(init);
577                      addlit(yylvalstr, yytextlen);
578                      if (literallen == 0) returni (error);
579                      if (literallen >= namedatalen)
580                          {
581                              setlengthofliteralbuf(namedatalen);
582                              literallen = namedatalen;
583                          }
584                      yylvalstr = litbufdup();    
585                      returni (TBaseType.dax_column_token);
586                  }
587
588          break;
589          }
590
591
592  case 17:
593                               
594          {
595              start(xdbracket);
596              startlit();
597              addlit(yylvalstr, yytextlen);
598              break;
599          }
600        
601  case 18:
602                              
603          {
604              dummych1 = get_char();
605              unget_char(dummych1);
606              if (dummych1 == (char)10)
607              {
608                  dummych1 = get_char();
609                  addlit(yylvalstr+dummych1, yytextlen+1);
610              }
611              else
612                  addlit(yylvalstr, yytextlen);
613                  
614            
615
616          break;
617          }
618
619
620  case 19:
621                
622        {
623       returnc(yylvalstr.charAt(0));
624       break;
625        }
626        
627  case 20:
628                        
629        {
630       returnc(yylvalstr.charAt(0));
631       break;
632        }
633        
634  case 21:
635                
636        {
637                returni(TBaseType.less_equal);
638          break;
639        }
640
641  case 22:
642                
643        {
644                returni(TBaseType.great_equal);
645          break;
646        }
647        
648  case 23:
649                
650        {
651                returni(TBaseType.measure_equal);
652          break;
653        }
654        
655
656  case 24:
657                
658        {
659                returni(TBaseType.not_equal);
660          break;
661        }
662        
663  case 25:
664                
665        {
666                returnc(yylvalstr.charAt(0));
667          break;
668        }
669        
670  case 26:
671                
672        {
673                returni(TBaseType.logic_and_op);
674          break;
675        }
676        
677  case 27:
678                
679        {
680                returni(TBaseType.logic_or_op);
681          break;
682        }
683
684
685  case 28:
686                
687        {
688                returnc(yylvalstr.charAt(0));
689          break;
690        }
691
692
693  case 29:
694                
695          {
696            returni(lexnewline);
697            break;
698          }
699
700  case 30:
701        
702          {
703              returni(lexspace);
704              break;
705          }
706
707  case 31:
708                
709          {
710          if ((getyysstate() == xq)
711              || (getyysstate() == xd)
712              || (getyysstate() == xc)
713              || (getyysstate() == xdbracket))
714          {
715              addlit(yylvalstr, 1);
716              yyless(1);
717              return;//exit;
718          }
719
720          returni(cmtdoublehyphen);
721          break;
722          }
723
724
725
726
727  case 32:
728                        
729          {
730               returni(iconst);
731               break;
732          }
733
734  case 33:
735                        
736          {
737            ///*  for i in 1..5 loop, we can't recognize 1. as a decimal,but 1 as decimal
738            nchars = yytextlen;
739            if (yylvalstr.charAt(nchars-1) == '.')
740              {
741                dummych1 = get_char();
742                unget_char(dummych1);
743                if (dummych1 == '.')
744                  {
745                      yyless(nchars-1);
746                    yylvalstr = yylvalstr.substring(0,nchars - 1);
747                    returni (iconst);
748                    return;//exit;
749                  }
750              }
751            returni (fconst);
752           break;
753          }
754
755  case 34:
756                        
757          {
758           returni (fconst);
759           break;
760          }
761
762  case 35:
763              
764          {
765            int rw;
766            if ( (rw = iskeyword(yylvalstr)) != -1)   { returni(rw);}
767            else
768               returni(ident);
769            break;
770          }
771
772  case 36:
773           
774          {
775            returni(variable);
776
777            break;
778          }
779    
780  case 37:
781                        
782          {
783            returni( error);
784            break;
785          }
786    default:{
787     System.out.println("fatal error in yyaction");
788    }
789   }//switch
790}/*yyaction*/;
791
792
793
794        }