001package gudusoft.gsqlparser;
002
003import gudusoft.gsqlparser.nodes.TObjectName;
004import gudusoft.gsqlparser.nodes.TParseTreeNode;
005
006import java.util.Stack;
007
008/**
009 * Represents a source token which is the basic syntactic unit of SQL.
010 * A token can be a key word, an identifier, a quoted identifier, a literal (or constant), or a special character symbol.
011 * Tokens are normally separated by whitespace (space, tab, newline), but need not be if there is no ambiguity
012 * (which is generally only the case if a special character is adjacent to some other token type).
013 * <p>
014 * The parse tree node consists of source tokens.
015 * <p>
016 * <br>A list of source token will be available after parse or tokenize the input SQL.
017 * <pre>
018 * {@code
019 *
020 *   TGSqlParser sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
021 *   sqlparser.sqltext = "select col from t";
022 *   int ret = sqlparser.parse();
023 *   if (ret == 0){
024 *    for(int i=0;i<sqlparser.sourcetokenlist.size();i++){
025 *      TSourceToken st =  sqlparser.sourcetokenlist.get(i);
026 *      System.out.println(st.tokentype.toString()+" "+st.toString());
027 *    }
028 *   }else{
029 *     System.out.println(sqlparser.getErrormessage());
030 *   }
031 *
032 * }
033 * </pre>
034 * Get a list of source tokens after call the method {@link gudusoft.gsqlparser.TGSqlParser#parse() }
035 * or just call {@link gudusoft.gsqlparser.TGSqlParser#tokenizeSqltext()} if you only
036 * need to access tokens of input SQL without generating the full parse tree.
037 * <br><br>
038 * {@link #tokencode} is the unique id represents the type of token,
039 * some typical tokens are: whitespace, return, keyword, identifier. This value is mainly used by the parser internally.
040 * <p>
041 * {@link #tokentype} uniquely identify the token type in a more meaningful way. It's more easier to use
042 * this field in your program than tokencode.
043 */
044
045public class TSourceToken implements Cloneable {
046
047    public TSourceToken clone(){
048        TSourceToken cloneObject = new TSourceToken();
049        cloneObject.tokencode = this.tokencode;
050        cloneObject.tokenstatus = this.tokenstatus;
051        cloneObject.tokentype = this.tokentype;
052        cloneObject.dbObjectType = this.dbObjectType;
053        cloneObject.setAstext(this.getAstext());
054        cloneObject.lineNo = this.lineNo;
055        cloneObject.columnNo = this.columnNo;
056
057        return cloneObject;
058    }
059
060    public ESqlClause location = ESqlClause.unknown;
061
062    public static void  concatInChain(TSourceToken st1, TSourceToken st2){
063        st1.setNextTokenInChain(st2);
064        st2.setPrevTokenInChain(st1);
065    }
066
067    public void insertANewTokenAfterMe(TSourceToken newToken){
068        newToken.setNextTokenInChain(this.getNextTokenInChain());
069        if (this.getNextTokenInChain() != null){
070            this.getNextTokenInChain().setPrevTokenInChain(newToken);
071        }
072
073        newToken.setPrevTokenInChain(this);
074        this.setNextTokenInChain(newToken);
075    }
076
077    public void insertANewTokenBeforeMe(TSourceToken newToken){
078        newToken.setPrevTokenInChain(this.getPrevTokenInChain());
079        if (this.getPrevTokenInChain() != null){
080            this.getPrevTokenInChain().setNextTokenInChain(newToken);
081        }
082
083        newToken.setNextTokenInChain(this);
084        this.setPrevTokenInChain(newToken);
085    }
086
087    public  void updateNodeEndWithThisToken(){
088        for(int i=0;i<this.getNodesEndWithThisToken().size();i++){
089            TParseTreeNode node = this.getNodesEndWithThisToken().get(i);
090                if (this.getPrevTokenInChain() != null){
091                    node.setEndTokenDirectly(this.getPrevTokenInChain());
092                    this.getPrevTokenInChain().getNodesEndWithThisToken().push(node);
093                }
094        }
095    }
096
097    public void updateNodeStartWithThisToken(){
098        for(int i=0;i<this.getNodesStartFromThisToken().size();i++){
099            TParseTreeNode node = this.getNodesStartFromThisToken().get(i);
100            if (this.getNextTokenInChain() != null){
101                node.setStartTokenDirectly(this.getNextTokenInChain());
102                this.getNextTokenInChain().getNodesStartFromThisToken().push(node);
103            }
104        }
105    }
106
107    public void removeFromChain(){
108        if (this.getPrevTokenInChain() != null){
109            this.getPrevTokenInChain().setNextTokenInChain(this.getNextTokenInChain());
110            if (this.getNextTokenInChain() != null){
111                this.getNextTokenInChain().setPrevTokenInChain(this.getPrevTokenInChain());
112            }
113        }
114    }
115    private TSourceToken prevTokenInChain = null, nextTokenInChain = null;
116
117    public void setPrevTokenInChain(TSourceToken prevTokenInChain) {
118        this.prevTokenInChain = prevTokenInChain;
119    }
120
121    public void setNextTokenInChain(TSourceToken nextTokenInChain) {
122        this.nextTokenInChain = nextTokenInChain;
123    }
124
125    public TSourceToken getPrevTokenInChain() {
126        return prevTokenInChain;
127    }
128
129    public TSourceToken getNextTokenInChain() {
130        return nextTokenInChain;
131    }
132
133    public ETokenStatus getTokenstatus() {
134        return tokenstatus;
135    }
136
137    public int getQuoteSymbolLength(){
138        String pstr = this.toString();
139        if (pstr.startsWith("'")){
140            return 1;
141        }else if (pstr.startsWith("$")){
142            return this.dolqstart.length();
143        }else {
144            return 0;
145        }
146    }
147    public String getQuotedString(){
148        String pstr = this.toString();
149        if (pstr.startsWith("'")){
150            return pstr.substring(1,pstr.length()-1);
151        }else if (pstr.startsWith("$")){
152            return pstr.substring(this.dolqstart.length(),pstr.length()-  this.dolqstart.length());
153        }else {
154            return "";
155        }
156    }
157
158    /**
159     * String literal in SQL usually inside ``.
160     * Quoted identifier in SQL usually surrounded by [ and ],
161     * This method only returns the text inside those surroundings.
162     *
163     * @return text inside `` and []
164     */
165    public String getTextWithoutQuoted(){
166        if ((toString().startsWith("'"))||(toString().startsWith("["))||(toString().startsWith("\""))){
167            return  toString().substring(1, toString().length() - 1);
168        }else
169          return  toString();
170    }
171
172    /**
173     * Class constructor
174     */
175    public  TSourceToken(){
176    }
177
178    /**
179     * Class constructor, set a string value.
180     *
181     * @param s the string value this toke represent for.
182     */
183    public TSourceToken(String s){
184        setAstext(s);
185    }
186
187    /**
188     * The string text of this token
189     *
190     * @return the string text of this token
191     */
192    public String toScript(){
193        return getAstext();
194    }
195
196
197    public int prevTokenCode = 0;
198    /**
199     * Unique id of this token used by parser internally.
200     * check available value start from {@link gudusoft.gsqlparser.TBaseType#cmtslashstar}
201     */
202    public int tokencode;
203
204    /**
205     * the line number of the first character in this token
206     */
207    public long lineNo;
208
209    /**
210     * the column number of the first character in this token
211     */
212    public long columnNo;
213
214    /**
215     * Token's offset from the beginning of the input query.
216     * <pre>
217     *     {@code
218     *    public void testOffset(){
219     *        TGSqlParser sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
220     *        sqlparser.sqltext = "select f from t\n" +
221     *                "where f>1\n";
222     *        assertTrue(sqlparser.parse() == 0);
223     *        for (int i=0;i<sqlparser.sourcetokenlist.size();i++){
224     *            TSourceToken st = sqlparser.sourcetokenlist.get(i);
225     *            String textFromOffset = sqlparser.sqltext.toString().substring((int)st.offset,(int)st.offset+st.toString().length());
226     *            assertTrue(st.toString().equalsIgnoreCase(textFromOffset));
227     *        }
228     *    }
229     *     }
230     * </pre>
231     */
232    public long offset;
233
234    /**
235     * Uniquely identify the token type in a more meaningful way. It's more easier to use
236     * this field in your program than tokencode.
237     * check available value in {@link gudusoft.gsqlparser.ETokenType ETokenType}
238     */
239    public ETokenType tokentype;
240
241    /**
242     * Container for this token which is a list of source token, this is the reference to {@link gudusoft.gsqlparser.TGSqlParser#sourcetokenlist}
243     */
244    public TSourceTokenList container;
245    
246    /**
247     * Index of this token in the {@link #container}, start from 0
248     *
249     * <pre>
250     *     {@code
251     *    public void testPosinList(){
252     *        TGSqlParser sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
253     *        sqlparser.sqltext = "select f from t\n" +
254     *                "where f>1\n";
255     *        assertTrue(sqlparser.parse() == 0);
256     *        for (int i=0;i<sqlparser.sourcetokenlist.size();i++){
257     *            assertTrue(i == sqlparser.sourcetokenlist.get(i).posinlist);
258     *        }
259     *    }
260     *     }
261     * </pre>
262     */
263    public int posinlist;
264
265    /**
266     * The text content of this token.
267     *
268     * <p>This public field is maintained for backwards compatibility with existing code.
269     * New code should use getter/setter methods instead of direct field access.</p>
270     *
271     * <h3>Recommended Usage:</h3>
272     * <ul>
273     *   <li><b>Reading text:</b> Use {@link #toString()} method</li>
274     *   <li><b>Writing text:</b> Use {@link #setAstext(String)} method</li>
275     * </ul>
276     *
277     * @see #toString() - Safe way to read token text
278     * @see #getAstext() - Alternative getter method
279     * @see #setAstext(String) - Proper way to set token text
280     * @since 1.0 (public field maintained for backwards compatibility)
281     */
282    public String astext;
283
284    public void appendText(TSourceToken st){
285
286        if (this.toString().startsWith(".")){
287            // dataset_id.3_ST_LEAK_ORDERS, .3_ST_LEAK_ORDERS 是由 .3 和_ST_LEAK_ORDERS组成
288            // 在拼接 .3 和_ST_LEAK_ORDERS 时,需要把 .3 中的 . 去掉
289            this.setAstext(this.toString().substring(1,this.toString().length()));
290            this.insertANewTokenBeforeMe(new TSourceToken("."));
291        }
292        if (st.toString().endsWith(".")){
293            this.setAstext(this.getAstext() +st.toString().substring(0,st.toString().length()-1));
294            //st.tokenstatus = ETokenStatus.tsdeleted; // make sure this token don't appear in toString()
295
296            // 以下代码确保 920778. 后面的 . 在 TObjectName.toString() 中出现
297            //  prod-gcp-data-lakehouse-920778.EXTERNALDATA.SMS_LOCATION
298
299            st.tokencode = '.';
300            st.setAstext(".");
301            st.tokentype = ETokenType.ttperiod;
302        }else{
303            this.setAstext(this.getAstext() +st.toString());
304            st.tokenstatus = ETokenStatus.tsdeleted;
305        }
306    }
307
308    public void appendText(String text){
309        this.setAstext(this.getAstext() +text);
310    }
311
312    public void insertText(TSourceToken st){
313        this.setAstext(st.toString()+ this.getAstext());
314        st.tokenstatus = ETokenStatus.tsdeleted;
315    }
316    public void setTextWithBackup(String newText){
317
318        prevsourcecode = getAstext();
319        tag = TBaseType.tag_token_value_changed_in_on_canonical;
320
321        setAstext(newText);
322    }
323
324    public boolean isChangedInAsCanonical(){
325        return tag == TBaseType.tag_token_value_changed_in_on_canonical;
326    }
327
328    public void setTokenstatus(ETokenStatus tokenstatus) {
329        this.tokenstatus = tokenstatus;
330    }
331
332    // private ETokenStatus statusForRewrite = ETokenStatus.tsoriginal;
333
334    /**
335     * Maintenance the status of this token during lex and parsing.
336     * Used by the parser internally.
337     */
338    public ETokenStatus tokenstatus;
339
340    /**
341     * Start part of Dollar-quoted String Constants of PostgreSQL.
342     *
343     * While the standard syntax for specifying string constants is usually convenient,
344     * it can be difficult to understand when the desired string contains many single quotes or backslashes,
345     * since each of those must be doubled. To allow more readable queries in such situations,
346     * PostgreSQL provides another way, called "dollar quoting", to write string constants.
347     * A dollar-quoted string constant consists of a dollar sign ($), an optional "tag" of zero or more characters,
348     * another dollar sign, an arbitrary sequence of characters that makes up the string content, a dollar sign,
349     * the same tag that began this dollar quote, and a dollar sign.
350     * <p></p>
351     * For example, here are two different ways to specify the string "Dianne's horse" using dollar quoting:
352     * <pre>
353     *     $$Dianne's horse$$
354     *     $SomeTag$Dianne's horse$SomeTag$
355     * </pre>
356     *
357     * This field will return $$ and $SomeTag$ accordingly.
358     */
359    public String  dolqstart;//postgresql, start part of Dollar-quoted String Constants
360
361    private String prevsourcecode;
362
363    public void restoreText(){
364        setAstext(prevsourcecode);
365        tag = 0;
366    }
367
368    public boolean insqlpluscmd;
369
370    /*
371     * source token can be end token of one or more parse tree nodes
372     * NodesEndWithThisToken includes all those parse tree nodes
373     */
374    private Stack<TParseTreeNode> NodesEndWithThisToken = null;
375
376    /**
377     *
378     *
379     * A list of nodes whose end token is this token.
380     *
381     * @return A list of nodes whose end token is this token.
382     */
383    public Stack<TParseTreeNode> getNodesEndWithThisToken() {
384        if (this.NodesEndWithThisToken == null){
385            this.NodesEndWithThisToken = new Stack<TParseTreeNode>();
386        }
387        return NodesEndWithThisToken;
388    }
389
390    /*
391     * source token can be start token of one or more parse tree nodes,
392     * NodesStartFromThisToken includes those parse tree nodes.
393     */
394    private Stack<TParseTreeNode> NodesStartFromThisToken = null;
395
396    /**
397     *
398     * A list of node whose start token is this token
399     *
400     * @return A list of node whose start token is this token
401     */
402    public Stack<TParseTreeNode> getNodesStartFromThisToken() {
403        if (this.NodesStartFromThisToken == null){
404            this.NodesStartFromThisToken = new Stack<TParseTreeNode>();
405        }
406        return NodesStartFromThisToken;
407    }
408
409
410    /**
411     * SQL statement that owns this token.
412     */
413    public TCustomSqlStatement stmt;
414
415    /**
416     * Space to save a value for temporary use
417     */
418    public int tag;
419
420    /**
421     * @deprecated use {@link #setDbObjectType} instead.
422     *
423     * @param dbObjType the database object type
424     */
425    public void setDbObjType(int dbObjType) {
426        this.dbObjType = dbObjType;
427    }
428
429    /**
430     * Token in a {@link TObjectName} has the same database object type as the objectName.
431     * Please use {@link gudusoft.gsqlparser.nodes.TObjectName#getDbObjectType} instead of this method if possible.
432     *
433     * @return the type of the database object
434     */
435    public int getDbObjType() {
436
437        return dbObjType;
438    }
439
440    /**
441     * Set the database object type of this token
442     *
443     * @param dbObjectType database object type
444     */
445    public void setDbObjectType(EDbObjectType dbObjectType) {
446        this.dbObjectType = dbObjectType;
447    }
448
449    /**
450     * Token in a {@link TObjectName} has the same database object type as the objectName.
451     * Please use {@link gudusoft.gsqlparser.nodes.TObjectName#getDbObjectType} instead of this method if possible.
452     *
453     * @return the type of the database object
454     */
455    public EDbObjectType getDbObjectType() {
456
457        return dbObjectType;
458    }
459
460    private EDbObjectType dbObjectType = EDbObjectType.unknown;
461    private int dbObjType = TObjectName.ttobjUnknown;
462
463
464
465    /**
466     * The database vendor which the SQL script includes this token will run against
467     *
468     * @param dbvendor the database vendor such as Oracle, DB2 and so on.
469     */
470    public void setDbvendor(EDbVendor dbvendor) {
471        this.dbvendor = dbvendor;
472    }
473
474    /**
475     * The database vendor which the SQL script includes this token will run against
476     * @return dbvendor the database vendor such as Oracle, DB2 and so on.
477     */
478    public EDbVendor getDbvendor() {
479
480        return dbvendor;
481    }
482
483    private EDbVendor dbvendor;
484
485    /**
486     * set new string of this token
487     *
488     * @param str the new string text
489     */
490    public void setString(String str){
491        setAstext(str);
492    }
493
494    /**
495     * The original string text for this token.
496     * @return  the string text
497     */
498    public String toString(){
499        return astext != null ? astext : "";
500    }
501    
502    
503
504    /**
505     * String text with the debug information such as coordinate, token code, token type
506     *
507     * @return the string value with full debug info
508     */
509    public String toStringDebug(){
510      String ret = lineNo +","+ columnNo +","+ getAstext().length()+","+offset+","+tokencode+" "+tokentype;
511      if (tokencode == TBaseType.cmtslashstar)
512        {ret = ret +" multi line comment";}
513      else if (tokencode == TBaseType.cmtdoublehyphen)
514        {ret = ret +" single line comment";}
515      else if (tokencode == TBaseType.lexspace)
516        {ret = ret +" space"; }
517      else if (tokencode == TBaseType.lexnewline)
518        {ret = ret +" newline"; }
519      else
520        {ret = ret +" "+ getAstext();}
521      return ret;
522    }
523
524    /**
525     * Space, return, comments are treated as non-solid token by default
526     *
527     * @param tokentype token type
528     * @return true if token type is not one of ttwhitespace,ttreturn,ttsimplecomment,ttbracketedcomment
529     */
530    public  static boolean isnonsolidtoken(ETokenType tokentype){
531        return ( (tokentype == ETokenType.ttwhitespace) || (tokentype == ETokenType.ttreturn)
532                ||(tokentype == ETokenType.ttsimplecomment)||(tokentype == ETokenType.ttbracketedcomment));
533    }
534
535    /**
536     *  Is this token a solid token or not.
537     *
538     * @return true if it's a non-solid token.
539     */
540    public  boolean isnonsolidtoken(){
541        return !issolidtoken();
542    }
543
544    /**
545     * Is this token a non-solid token or not.
546     *
547     * @return true if it's a solid token.
548     */
549    public boolean issolidtoken(){
550        return !isnonsolidtoken(this.tokentype);
551    }
552
553    private TSourceTokenList tokensBefore = null;
554    private TSourceTokenList tokensAfter = null;
555
556    /**
557     * Used in sql formatter package only.
558     *
559     * @return source token list
560     */
561    public TSourceTokenList getTokensAfter() {
562        if (this.tokensAfter == null){
563            this.tokensAfter = new TSourceTokenList();
564        }
565        return tokensAfter;
566    }
567
568    /**
569     * Used in sql formatter package only
570     *
571     * @return source token list
572     */
573    public TSourceTokenList getTokensBefore() {
574        if (this.tokensBefore == null){
575            this.tokensBefore = new TSourceTokenList();
576        }
577        return tokensBefore;
578    }
579
580    private TSourceToken replaceToken = null;
581
582    /**
583     * Used in sql formatter package only
584     *
585     * @param replaceToken replaced token
586     */
587    public void setReplaceToken(TSourceToken replaceToken) {
588        this.replaceToken = replaceToken;
589    }
590
591    /**
592     * Used in sql formatter package only
593     *
594     * @return replaced token
595     */
596    public TSourceToken getReplaceToken() {
597
598        return replaceToken;
599    }
600
601    public  TSourceToken nextToken(){
602        TSourceToken ret = null;
603        if (this.container == null) return ret;
604        if (this.posinlist >= this.container.size() - 1) return null;
605        return this.container.get(this.posinlist+1);
606    }
607
608
609    public TSourceToken searchTokenAtTheEndOfSameLine(){
610        TSourceToken ret = null;
611        if (this.container == null) return ret;
612        int i = this.container.searchLastTokenAtTheSameLine(this.posinlist);
613        if (i == -1) return null;
614        return this.container.get(i);
615    }
616    /**
617     * Search a token before or after this token in the same source token list.
618     * The result token should has the tokencode equals to the targetTokenCode
619     * in a specified range.
620     *
621     * @param targetTokenCode, the token code need to be searched
622     * @param range, &gt; 0, search token start from the next token and forward,
623     * = 0, just compare with this token,
624     * &lt; 0, search from the previous token and backword.
625     * @return the token with the same token code, otherwise, return null.
626     */
627    public TSourceToken searchToken(int targetTokenCode,int range, int stopTokenCode, boolean stopAtSemiColon){
628        TSourceToken ret = null;
629        if (this.container == null) return ret;
630        return this.container.searchToken(targetTokenCode,"",this,range,stopTokenCode,stopAtSemiColon);
631    }
632
633    public TSourceToken searchToken(int targetTokenCode,int range){
634        return searchToken(targetTokenCode,range,0,false);
635    }
636
637    /**
638     * Search a token before or after this token in the same source token list.
639     * The result token should has the string text equals to the targetTokenText
640     * in a specified range.
641     *
642     * @param targetTokenText, the target string text
643     * @param range, &gt; 0, search token start from the next token and forward,
644     * = 0, just compare with this token,
645     * &lt; 0, search from the previous token and backword.
646     * @return the token with the same token code, otherwise, return null.
647     */
648    public TSourceToken searchToken(String targetTokenText,int range, int stopTokenCode, boolean stopAtSemiColon){
649        TSourceToken ret = null;
650        if (this.container == null) return ret;
651        return this.container.searchToken(0,targetTokenText,this,range,stopTokenCode,stopAtSemiColon);
652    }
653
654    public TSourceToken searchToken(String targetTokenText,int range) {
655            return searchToken(targetTokenText,range,0,false);
656    }
657    /**
658     * Search the first non-solid token after the next objectName.
659     * <p></p>
660     * Take this SQL for example:
661     * <pre>
662     * return new scott.func(x1);
663     * </pre>
664     * If this token is <code>new</code>, then call searchTokenAfterObjectName will return <code>(</code> token.
665     *
666     * @return solid token after the next objectName
667     */
668    public TSourceToken searchTokenAfterObjectName(){
669        TSourceToken ret = null;
670        if (this.container == null) return ret;
671        int i = container.nextObjectNameToken(posinlist,1,false);
672        if (i == -1) return  ret;
673        return container.get(i).nextSolidToken();
674    }
675
676    public  TSourceToken nextSolidToken(int pstep){
677        TSourceToken ret = null;
678        if (this.container == null) return ret;
679        return this.container.nextsolidtoken(this,pstep,false);
680    }
681
682
683    /**
684     * The next token whose {@link #tokentype} is not ttreturn,ttwhitespace,ttsimplecomment and ttbracketedcomment.
685     *
686     * @return the next solid token, returns null if not found.
687     */
688    public  TSourceToken nextSolidToken(){
689        TSourceToken ret = null;
690        if (this.container == null) return ret;
691        return this.container.nextsolidtoken(this,1,false);
692    }
693
694    /**
695     * The next token whose {@link #tokentype} is not ttreturn,ttwhitespace,ttsimplecomment and ttbracketedcomment.
696     *
697     * @param treatCommentAsSolidToken, set to true will treat comment token as a solid token
698     * @return the solid token if found, otherwise, return null
699     */
700    public  TSourceToken nextSolidToken(boolean treatCommentAsSolidToken){
701        TSourceToken ret = null;
702        if (this.container == null) return ret;
703        return this.container.nextsolidtoken(this,1,treatCommentAsSolidToken);
704    }
705
706
707    /**
708     * The previous token whose {@link #tokentype} is not ttreturn,ttwhitespace,ttsimplecomment and ttbracketedcomment.
709     *
710     * @return the solid token if found, otherwise, return null
711     */
712    public  TSourceToken prevSolidToken(){
713        TSourceToken ret = null;
714        if (this.container == null) return ret;
715        return this.container.nextsolidtoken(this,-1,false);
716    }
717
718    /**
719     * The previous token whose {@link #tokentype} is not ttreturn,ttwhitespace,ttsimplecomment and ttbracketedcomment
720     *
721     * @param treatCommentAsSolidToken set to true will treat comment token as a solid token
722     * @return the solid token if found, otherwise, return null
723     */
724    public  TSourceToken prevSolidToken(boolean treatCommentAsSolidToken){
725        TSourceToken ret = null;
726        if (this.container == null) return ret;
727        return this.container.nextsolidtoken(this,-1,treatCommentAsSolidToken);
728    }
729
730    /**
731     * Check to see if this token is the first token in a line of the input SQL
732     *
733     * @return true if this is the first token of a line, otherwise, return false
734     */
735    public boolean isFirstTokenOfLine(){
736        TSourceToken st = prevSolidToken();
737        if (st == null) return true;
738        return  (st.lineNo != this.lineNo);
739    }
740
741    /**
742     * Check to see if this token is the last token of in a line in the input SQL
743     *
744     * @return true if this is the last token of a line, otherwise, return false
745     */
746    public boolean isLastTokenOfLine(){
747        TSourceToken st = nextSolidToken();
748        if (st == null) return true;
749        return  (st.lineNo != this.lineNo);
750    }
751
752
753    private TSourceToken linkToken = null;
754
755    /**
756     * Create a link between two tokens. Make it easy to access another linked token.
757     * Usually, those are two tokens like the parenthesis in this SQL: <code>(select * from t)</code>
758     *
759     * @param linkToken the token need to be linked
760     */
761    public void setLinkToken(TSourceToken linkToken) {
762        this.linkToken = linkToken;
763        linkToken.linkToken = this;
764    }
765
766    /**
767     * Gets the linked token.
768     * Take this SQL for example:
769     * <code>(select * from t)</code>, if this token is '(', then you call this method will return ')' token.
770     *
771     * @return the paired parenthesis in select, expression
772     */
773    public TSourceToken getLinkToken() {
774
775        return linkToken;
776    }
777
778    /**
779     * Remove double quote <code>""</code>, bracket quote <code>[]</code> , left/right brace <code>{}</code>
780     * from a delimited identifier and return string text of this identifier.
781     *
782     * @return string text of this token
783     *
784     * @deprecated since 2.5.3.4
785     */
786    public String toUnQuotedString(){
787        if ((tokentype == ETokenType.ttdqstring)
788                ||(tokentype == ETokenType.ttdbstring)
789                ||(tokentype == ETokenType.ttbrstring)
790        ){
791                  return toString().substring(1,toString().length() - 1);
792        }else if ((dbvendor == EDbVendor.dbvmysql)&&(toString().startsWith("`"))){
793            return toString().substring(1,toString().length() - 1);
794        }else {
795            return toString();
796        }
797    }
798
799    /**
800     * Text representation for this token.
801     */
802    public String getAstext() {
803        return  toString(); //astext;
804    }
805
806    public void setAstext(String astext) {
807        this.astext = astext;
808    }
809}