Source code

001
002package gudusoft.gsqlparser.pp.utils;
003
004import gudusoft.gsqlparser.ETokenStatus;
005import gudusoft.gsqlparser.ETokenType;
006import gudusoft.gsqlparser.TSourceToken;
007import gudusoft.gsqlparser.TSourceTokenList;
008import gudusoft.gsqlparser.pp.logger.PPLogger;
009import gudusoft.gsqlparser.pp.para.GFmtOpt;
010
011/**
012 * the helper class used for operating the source token
013 * 
014 * @author zhoujun
015 * 
016 */
017public class SourceTokenOperator
018{
019
020        public static int removeWhitespaceBackward( GFmtOpt opt,
021                        TSourceTokenList list, int end )
022        {
023                if ( end == 0 )
024                {
025                        return 0;
026                }
027
028                StringBuilder buf = new StringBuilder( );
029                buf.append( "removeWhitespaceAndReturnFromEnd:\ntargetToken:" )
030                                .append( list.get( end ).astext )
031                                .append( " newToken:" )
032                                .append( list.get( end ).astext.replaceAll( "\\n", "\\\\n" ) )
033                                .append( "(type:" )
034                                .append( list.get( end ).tokentype )
035                                .append( ")" );
036                int deleteWhitespaceCount = 0;
037                // int deleteReturn = 0;
038
039                int pos = end - 1;
040
041                // remove all white space before end item
042                TSourceToken token = list.get( pos );
043                while ( token.tokentype == ETokenType.ttwhitespace )
044                {
045
046                        if ( token.tokentype == ETokenType.ttwhitespace )
047                        {
048                                deleteWhitespaceCount++;
049                        }
050
051                        if ( opt.opearateSourceToken )
052                        {
053                                token.setReplaceToken( SourceTokenConstant.EMPTY );
054                        }
055
056                        pos--;
057                        if ( pos >= 0 )
058                        {
059                                token = token.container.get( pos );
060                        }
061                        else
062                        {
063                                break;
064                        }
065                }
066
067                if ( token.tokentype == ETokenType.ttreturn )
068                {
069                        StringBuilder b = new StringBuilder( );
070                        for ( int i = 0; i < token.astext.length( ); i++ )
071                        {
072                                char c = token.astext.charAt( i );
073                                if ( c != ' ' )
074                                {
075                                        b.append( c );
076                                }
077                        }
078                        token.astext = b.toString( );
079                }
080
081                buf.append( "\nDeleteWhiteSpace:" ).append( deleteWhitespaceCount );
082                PPLogger.info( 2, buf.toString( ), null );
083
084                return pos;
085        }
086
087        /**
088         * remove the white space and the return token from the end
089         * 
090         * @param list
091         * @param end
092         * @return the pos which is not white space and return token in the source
093         *         token list
094         */
095        public static int removeWhitespaceAndReturnFromEnd( GFmtOpt opt,
096                        TSourceTokenList list, int end )
097        {
098                if ( end == 0 )
099                {
100                        return 0;
101                }
102
103                StringBuilder buf = new StringBuilder( );
104                buf.append( "removeWhitespaceAndReturnFromEnd:\ntargetToken:" )
105                                .append( list.get( end ).astext )
106                                .append( " newToken:" )
107                                .append( list.get( end ).astext.replaceAll( "\\n", "\\\\n" ) )
108                                .append( "(type:" )
109                                .append( list.get( end ).tokentype )
110                                .append( ")" );
111                int deleteWhitespaceCount = 0;
112                int deleteReturn = 0;
113
114                int pos = end - 1;
115
116                // remove all white space before end item
117                TSourceToken token = list.get( pos );
118                while ( token.tokentype == ETokenType.ttwhitespace
119                                || token.tokentype == ETokenType.ttreturn )
120                {
121                        if ( token.tokentype == ETokenType.ttwhitespace )
122                        {
123                                deleteWhitespaceCount++;
124                        }
125
126                        if ( token.tokentype == ETokenType.ttreturn )
127                        {
128                                deleteReturn++;
129                        }
130
131                        if ( opt.opearateSourceToken )
132                        {
133                                token.setReplaceToken( SourceTokenConstant.EMPTY );
134                        }
135
136                        pos--;
137                        if ( pos >= 0 )
138                        {
139                                token = token.container.get( pos );
140                        }
141                        else
142                        {
143                                break;
144                        }
145                }
146
147                buf.append( "\nDeleteWhiteSpace:" )
148                                .append( deleteWhitespaceCount )
149                                .append( ", DeleteReturn:" )
150                                .append( deleteReturn );
151                PPLogger.info( 2, buf.toString( ), null );
152
153                return pos;
154        }
155
156        /**
157         * remove the white space and the return token from the end
158         * 
159         * @param token
160         * @return the pos which is not whitespace and return token in the source
161         *         token list
162         */
163        public static int removeWhitespaceAndReturnFromEnd( GFmtOpt opt,
164                        TSourceToken token )
165        {
166                if ( token == null )
167                {
168                        return -1;
169                }
170                return removeWhitespaceAndReturnFromEnd( opt,
171                                token.container,
172                                token.posinlist );
173        }
174
175        /**
176         * remove the white space and the return token before this token and after
177         * this token
178         * 
179         * @param opt
180         * @param token
181         */
182        public static void removeWhitespaceAndReturnFormBeforeAndAfter(
183                        GFmtOpt opt, TSourceToken token )
184        {
185                if ( token == null )
186                {
187                        return;
188                }
189                removeWhitespaceAndReturnFromEnd( opt, token );
190                removeWhitespaceAndReturnFromStart( opt,
191                                token.container,
192                                token.posinlist + 1 );
193
194        }
195
196        /**
197         * remove the white space and the return token
198         * 
199         * @param list
200         * @param start
201         * @param end
202         */
203        public static void removeWhitespaceAndReturn( GFmtOpt opt,
204                        TSourceTokenList list, int start, int end )
205        {
206                StringBuilder buf = new StringBuilder( );
207                buf.append( "removeWhitespaceAndReturn:\ntargetToken:" )
208                                .append( list.get( end ).astext )
209                                .append( " newToken:" )
210                                .append( list.get( end ).astext.replaceAll( "\\n", "\\\\n" ) )
211                                .append( "(type:" )
212                                .append( list.get( end ).tokentype )
213                                .append( ")" );
214                int deleteWhitespaceCount = 0;
215                int deleteReturn = 0;
216
217                for ( int i = start; i < end; i++ )
218                {
219                        if ( i < 0 || i > list.size( ) )
220                        {
221                                return;
222                        }
223                        TSourceToken token = list.get( i );
224                        if ( token.tokentype == ETokenType.ttwhitespace
225                                        || token.tokentype == ETokenType.ttreturn )
226                        {
227
228                                if ( token.tokentype == ETokenType.ttwhitespace )
229                                {
230                                        deleteWhitespaceCount++;
231                                }
232
233                                if ( token.tokentype == ETokenType.ttreturn )
234                                {
235                                        deleteReturn++;
236                                }
237                                if ( opt.opearateSourceToken )
238                                {
239                                        token.setReplaceToken( SourceTokenConstant.EMPTY );
240                                }
241                        }
242                }
243
244                buf.append( "\nDeleteWhiteSpace:" )
245                                .append( deleteWhitespaceCount )
246                                .append( ", DeleteReturn:" )
247                                .append( deleteReturn );
248                PPLogger.info( 2, buf.toString( ), null );
249
250        }
251
252        /**
253         * remove the white space and the return token from the end
254         * 
255         * @param list
256         */
257        public static int removeWhitespaceAndReturnFromStart( GFmtOpt opt,
258                        TSourceTokenList list, int startPos )
259        {
260                StringBuilder buf = new StringBuilder( );
261                if ( list.size( ) <= startPos )
262                {
263                        return list.size( ) - 1;
264                }
265                buf.append( "removeWhitespaceAndReturnFromStart:\ntargetToken:" )
266                                .append( list.get( startPos ).astext )
267                                .append( " newToken:" )
268                                .append( list.get( startPos ).astext.replaceAll( "\\n", "\\\\n" ) )
269                                .append( "(type:" )
270                                .append( list.get( startPos ).tokentype )
271                                .append( ")" );
272                int deleteWhitespaceCount = 0;
273                int deleteReturn = 0;
274
275                int pos = startPos;
276                // remove all white space before end item
277                TSourceToken token = list.get( pos );
278                while ( token.tokentype == ETokenType.ttwhitespace
279                                || token.tokentype == ETokenType.ttreturn )
280                {
281                        if ( token.tokentype == ETokenType.ttwhitespace )
282                        {
283                                deleteWhitespaceCount++;
284                        }
285
286                        if ( token.tokentype == ETokenType.ttreturn )
287                        {
288                                deleteReturn++;
289                        }
290
291                        if ( opt.opearateSourceToken )
292                        {
293                                token.setReplaceToken( SourceTokenConstant.EMPTY );
294                        }
295
296                        pos++;
297                        if ( pos < list.size( ) )
298                        {
299                                token = token.container.get( pos );
300                        }
301                        else
302                        {
303                                break;
304                        }
305                }
306
307                buf.append( "\nDeleteWhiteSpace:" )
308                                .append( deleteWhitespaceCount )
309                                .append( ", DeleteReturn:" )
310                                .append( deleteReturn );
311                PPLogger.info( 2, buf.toString( ), null );
312
313                return pos;
314        }
315
316        /**
317         * calculate the text length for all virtual token (which are call before
318         * tokens, after tokens and replace token)
319         * 
320         * @param list
321         * @param start
322         * @param end
323         * @return
324         */
325        public static int textLengthVT( TSourceTokenList list, int start, int end )
326        {
327                int total = 0;
328                for ( int i = start; i < end; i++ )
329                {
330                        TSourceToken token = list.get( i );
331
332                        if ( token.getTokensBefore( ).size( ) > 0 )
333                        {
334                                total += textLengthVT( token.getTokensBefore( ),
335                                                0,
336                                                token.getTokensBefore( ).size( ) );
337                        }
338                        if ( token.getReplaceToken( ) != null )
339                        {
340                                total += ( token.getReplaceToken( ).astext != null ? token.getReplaceToken( ).astext.length( )
341                                                : 0 );
342                        }
343                        else
344                        {
345                                total += ( token.astext != null ? token.astext.length( ) : 0 );
346                        }
347                        if ( token.getTokensAfter( ).size( ) > 0 )
348                        {
349                                total += textLengthVT( token.getTokensAfter( ),
350                                                0,
351                                                token.getTokensAfter( ).size( ) );
352                        }
353                }
354
355                return total;
356        }
357
358        /**
359         * calcaulate the text length from a given position to the line start. VT is
360         * the abbr for "virtual token" (which contains before tokens, after tokens
361         * and replace token)
362         * 
363         * @param list
364         * @param pos
365         * @return
366         */
367        public static int curColumnNumberVT( TSourceTokenList list, int pos )
368        {
369
370                int total = 0;
371
372                for ( int i = pos; i >= 0; i-- )
373                {
374                        TSourceToken token = list.get( i );
375                        
376                        if(token.tokenstatus == ETokenStatus.tsdeleted) {
377                                continue;
378                        } 
379
380                        // the after tokens in the last sourceToken should not calculated.
381                        if ( i != pos && token.getTokensAfter( ).size( ) > 0 )
382                        {
383                                for ( int j = token.getTokensAfter( ).size( ) - 1; j >= 0; j-- )
384                                {
385                                        TSourceToken after = token.getTokensAfter( ).get( j );
386                                        if ( after.tokentype != ETokenType.ttreturn )
387                                        {
388                                                total += ( after.astext != null ? after.astext.length( )
389                                                                : 0 );
390                                        }
391                                        else
392                                        {
393                                                return total;
394                                        }
395                                }
396                        }
397                        if ( i != pos )
398                        {
399                                if ( token.getReplaceToken( ) != null )
400                                {
401                                        if ( token.getReplaceToken( ).tokentype != ETokenType.ttreturn )
402                                        {
403                                                total += ( token.getReplaceToken( ).astext != null ? token.getReplaceToken( ).astext.length( )
404                                                                : 0 );
405                                        }
406                                        else
407                                        {
408                                                return total;
409                                        }
410                                }
411                                else
412                                {
413                                        if ( token.tokentype != ETokenType.ttreturn )
414                                        {
415                                                if(token.tokentype == ETokenType.ttsimplecomment) {
416                                                        if(token.nextToken()!=null &&
417                                                                        token.nextToken().astext!=null &&
418                                                                        token.nextToken().astext.trim().length() == 0) {
419                                                                total -= token.nextToken().astext.length( );
420                                                        }
421                                                        continue;
422                                                }
423                                                if(isOracleHint(token)) {
424                                                        if(token.nextToken()!=null &&
425                                                                        token.nextToken().astext!=null &&
426                                                                        token.nextToken().astext.trim().length() == 0) {
427                                                                total -= token.nextToken().astext.length( );
428                                                        }
429                                                        continue;
430                                                }
431                                                total += ( token.astext != null ? token.astext.length( )
432                                                                : 0 );
433                                        }
434                                        else
435                                        {
436                                                return total;
437                                        }
438                                }
439                        }
440
441                        if ( token.getTokensBefore( ).size( ) > 0 )
442                        {
443                                for ( int j = token.getTokensBefore( ).size( ) - 1; j >= 0; j-- )
444                                {
445                                        TSourceToken before = token.getTokensBefore( ).get( j );
446                                        if ( before.tokentype != ETokenType.ttreturn )
447                                        {
448                                                total += ( before.astext != null ? before.astext.length( )
449                                                                : 0 );
450                                        }
451                                        else
452                                        {
453                                                return total;
454                                        }
455                                }
456                        }
457                }
458
459                return total;
460        }
461
462        private static boolean isOracleHint(TSourceToken token) {
463                return token!=null && token.astext!=null && token.astext.startsWith("/*+");
464        }
465
466        /**
467         * calcaulate the text length from a given position to the line start. VT is
468         * the abbr for "virtual token" (which contains before tokens, after tokens
469         * and replace token)
470         * 
471         * @param token
472         * @return
473         */
474        public static int curColumnNumberVT( TSourceToken token )
475        {
476                return curColumnNumberVT( token.container, token.posinlist );
477        }
478
479        /**
480         * calculate the current indent length VT is the abbr for "virtual token"
481         * (which contains before tokens, after tokens and replace token)
482         * 
483         * @param list
484         * @param pos
485         * @return
486         */
487        public static int curIndentLenVT( TSourceTokenList list, int pos )
488        {
489                int indentLen = 0;
490                for ( int i = pos; i >= 0; i-- )
491                {
492                        TSourceToken token = list.get( i );
493                        
494                        if(token.tokenstatus == ETokenStatus.tsdeleted) {
495                                continue;
496                        } 
497
498                        // the after tokens in the last sourceToken should not calculated.
499                        if ( i != pos && token.getTokensAfter( ).size( ) > 0 )
500                        {
501                                for ( int j = token.getTokensAfter( ).size( ) - 1; j >= 0; j-- )
502                                {
503                                        TSourceToken after = token.getTokensAfter( ).get( j );
504                                        if ( after.tokentype == ETokenType.ttreturn )
505                                        {
506                                                return indentLen;
507                                        }
508                                        else if ( after.tokentype == ETokenType.ttwhitespace )
509                                        {
510                                                indentLen += ( after.astext != null ? after.astext.length( )
511                                                                : 0 );
512                                        }
513                                        else
514                                        {
515                                                indentLen = 0;
516                                        }
517                                }
518                        }
519
520                        if ( i != pos )
521                        {
522                                if ( token.getReplaceToken( ) != null )
523                                {
524                                        if ( token.getReplaceToken( ).tokentype == ETokenType.ttreturn )
525                                        {
526                                                return indentLen;
527                                        }
528                                        else if ( token.getReplaceToken( ).tokentype == ETokenType.ttwhitespace )
529                                        {
530                                                indentLen += ( token.getReplaceToken( ).astext != null ? token.getReplaceToken( ).astext.length( )
531                                                                : 0 );
532                                        }
533                                        else
534                                        {
535                                                indentLen = 0;
536                                        }
537
538                                }
539                                else
540                                {
541                                        if ( token.tokentype == ETokenType.ttreturn )
542                                        {
543                                                return indentLen;
544                                        }
545                                        else if ( token.tokentype == ETokenType.ttwhitespace )
546                                        {
547                                                indentLen += ( token.astext != null ? token.astext.length( )
548                                                                : 0 );
549                                        }
550                                        else
551                                        {
552                                                indentLen = 0;
553                                        }
554                                }
555                        }
556
557                        if ( token.getTokensBefore( ).size( ) > 0 )
558                        {
559                                for ( int j = token.getTokensBefore( ).size( ) - 1; j >= 0; j-- )
560                                {
561                                        TSourceToken before = token.getTokensBefore( ).get( j );
562                                        if ( before.tokentype == ETokenType.ttreturn )
563                                        {
564                                                return indentLen;
565                                        }
566                                        else if ( before.tokentype == ETokenType.ttwhitespace )
567                                        {
568                                                indentLen += ( before.astext != null ? before.astext.length( )
569                                                                : 0 );
570                                        }
571                                        else
572                                        {
573                                                indentLen = 0;
574                                        }
575                                }
576                        }
577
578                }
579                return indentLen;
580        }
581
582        /**
583         * calculate the current indent length VT is the abbr for "virtual token"
584         * (which contains before tokens, after tokens and replace token)
585         * 
586         * @param token
587         * @return
588         */
589        public static int curIndentLenVT( TSourceToken token )
590        {
591                return curIndentLenVT( token.container, token.posinlist );
592        }
593
594        /**
595         * create the white space source token
596         * 
597         * @param length
598         * @return
599         */
600        public static TSourceToken createWhitespaceSourceToken( int length )
601        {
602                return WhitespaceSourceTokenHelper.createWhitespaceSourceToken( length );
603        }
604
605        private static TSourceToken noFormatFlag = null;
606        static
607        {
608                noFormatFlag = new TSourceToken( "" );
609                noFormatFlag.tokentype = ETokenType.ttwhitespace;
610        }
611
612        public static TSourceToken createNoFormatFlagToken( )
613        {
614                return noFormatFlag;
615        }
616
617        public static TSourceToken createReturnSourceToken( )
618        {
619                TSourceToken t = new TSourceToken( "\n" );
620                t.tokentype = ETokenType.ttreturn;
621                return t;
622        }
623
624        /**
625         * combine all the white spaces into a single white space
626         * 
627         * @param list
628         * @param startPos
629         * @param endPos
630         */
631        public static void combineWhitespace( TSourceTokenList list, int startPos,
632                        int endPos )
633        {
634                boolean findWhitespace = false;
635                for ( int i = startPos; i < endPos; i++ )
636                {
637                        if ( i >= 0 && i < list.size( ) )
638                        {
639                                TSourceToken token = list.get( i );
640                                
641                                if(token.tokenstatus == ETokenStatus.tsdeleted) {
642                                        continue;
643                                } 
644                                
645                                if ( token.tokentype == ETokenType.ttwhitespace )
646                                {
647                                        token.setReplaceToken( SourceTokenConstant.EMPTY );
648                                        findWhitespace = true;
649                                }
650                        }
651                }
652                if ( findWhitespace )
653                {
654                        list.get( endPos )
655                                        .getTokensBefore( )
656                                        .add( createWhitespaceSourceToken( 1 ) );
657                }
658        }
659
660        /**
661         * insert the source token before the target source token
662         * 
663         * @param targetToken
664         * @param needAddToken
665         */
666        public static void addBefore( GFmtOpt opt, TSourceToken targetToken,
667                        TSourceToken needAddToken )
668        {
669                if ( needAddToken == null )
670                {
671                        return;
672                }
673
674                StringBuilder buf = new StringBuilder( );
675                buf.append( "AddBefore:\ntargetToken:" )
676                                .append( targetToken.astext )
677                                .append( "(pos:" )
678                                .append( targetToken.posinlist )
679                                .append( ")" )
680                                .append( " newToken:" )
681                                .append( needAddToken.astext.replaceAll( "\\n", "\\\\n" ) )
682                                .append( "(type:" )
683                                .append( needAddToken.tokentype )
684                                .append( ")" );
685
686                if ( targetToken.getTokensBefore( ).size( ) > 0 )
687                {
688                        buf.append( "\noldTokenBefore:" );
689                        for ( int i = 0; i < targetToken.getTokensBefore( ).size( ); i++ )
690                        {
691                                TSourceToken token = targetToken.getTokensBefore( ).get( i );
692                                
693                                if(token.tokenstatus == ETokenStatus.tsdeleted) {
694                                        continue;
695                                } 
696                                
697                                buf.append( "[" )
698                                                .append( token.astext != null ? token.astext.replaceAll( "\\n",
699                                                                "\\\\n" )
700                                                                : "" )
701                                                .append( "] " )
702                                                .append( "(type:" )
703                                                .append( token.tokentype )
704                                                .append( ")" );
705                        }
706                }
707
708                PPLogger.info( 2, buf.toString( ), null );
709
710                if ( opt.opearateSourceToken )
711                {
712                        targetToken.getTokensBefore( ).add( needAddToken );
713                }
714        }
715
716        public static void addAfter( GFmtOpt opt, TSourceToken targetToken,
717                        TSourceToken needAddToken )
718        {
719                if ( needAddToken == null )
720                {
721                        return;
722                }
723
724                StringBuilder buf = new StringBuilder( );
725                buf.append( "AddAfter:\ntargetToken:" )
726                                .append( targetToken.astext )
727                                .append( "(pos:" )
728                                .append( targetToken.posinlist )
729                                .append( ")" )
730                                .append( " newToken:" )
731                                .append( needAddToken.astext != null ? needAddToken.astext.replaceAll( "\\n",
732                                                "\\\\n" )
733                                                : "" )
734                                .append( "(type:" )
735                                .append( needAddToken.tokentype )
736                                .append( ")" );
737
738                if ( targetToken.getTokensAfter( ).size( ) > 0 )
739                {
740                        buf.append( "\noldTokenAfter:" );
741                        for ( int i = 0; i < targetToken.getTokensAfter( ).size( ); i++ )
742                        {
743                                TSourceToken token = targetToken.getTokensAfter( ).get( i );
744                                
745                                if(token.tokenstatus == ETokenStatus.tsdeleted) {
746                                        continue;
747                                } 
748                                
749                                buf.append( "[" )
750                                                .append( token.astext != null ? token.astext.replaceAll( "\\n",
751                                                                "\\\\n" )
752                                                                : "" )
753                                                .append( "] " )
754                                                .append( "(type:" )
755                                                .append( token.tokentype )
756                                                .append( ")" );
757                        }
758                }
759
760                PPLogger.info( 2, buf.toString( ), null );
761
762                if ( opt.opearateSourceToken )
763                {
764                        targetToken.getTokensAfter( ).add( needAddToken );
765                }
766        }
767
768        /**
769         * remove all source token in the before source token list VT is the abbr
770         * for "virtual token" (that this method will operates before tokens, after
771         * tokens and replace token)
772         * 
773         * @param targetToken
774         */
775        public static void removeAllBeforeTokenVT( GFmtOpt opt,
776                        TSourceToken targetToken )
777        {
778                StringBuilder buf = new StringBuilder( );
779                buf.append( "RemoveAllBeforeTokenVT:" );
780                while ( targetToken.getTokensBefore( ).size( ) > 0 )
781                {
782                        TSourceToken token = targetToken.getTokensBefore( ).get( 0 );
783                        
784                        if(token.tokenstatus == ETokenStatus.tsdeleted) {
785                                continue;
786                        } 
787                        
788                        buf.append( "\nremoveToken:" )
789                                        .append( "[" )
790                                        .append( token.astext != null ? token.astext.replaceAll( "\\n",
791                                                        "\\\\n" )
792                                                        : "" )
793                                        .append( "] " )
794                                        .append( "(type:" )
795                                        .append( token.tokentype )
796                                        .append( ")" );
797                        if ( opt.opearateSourceToken )
798                        {
799                                targetToken.getTokensBefore( ).remove( 0 );
800                        }
801                }
802                PPLogger.info( 2, buf.toString( ), null );
803
804        }
805
806        public static void removeThisToken( TSourceToken token )
807        {
808                token.setReplaceToken( SourceTokenOperator.createWhitespaceSourceToken( 1 ) );
809        }
810}