001 002package gudusoft.gsqlparser.pp.utils; 003 004import gudusoft.gsqlparser.ETokenStatus; 005import gudusoft.gsqlparser.ETokenType; 006import gudusoft.gsqlparser.TSourceToken; 007import gudusoft.gsqlparser.TSourceTokenList; 008import gudusoft.gsqlparser.pp.logger.PPLogger; 009import gudusoft.gsqlparser.pp.para.GFmtOpt; 010 011/** 012 * the helper class used for operating the source token 013 * 014 * @author zhoujun 015 * 016 */ 017public class SourceTokenOperator 018{ 019 020 public static int removeWhitespaceBackward( GFmtOpt opt, 021 TSourceTokenList list, int end ) 022 { 023 if ( end == 0 ) 024 { 025 return 0; 026 } 027 028 StringBuilder buf = new StringBuilder( ); 029 buf.append( "removeWhitespaceAndReturnFromEnd:\ntargetToken:" ) 030 .append(list.get(end).getAstext()) 031 .append( " newToken:" ) 032 .append( list.get(end).getAstext().replaceAll( "\\n", "\\\\n" ) ) 033 .append( "(type:" ) 034 .append( list.get( end ).tokentype ) 035 .append( ")" ); 036 int deleteWhitespaceCount = 0; 037 // int deleteReturn = 0; 038 039 int pos = end - 1; 040 041 // remove all white space before end item 042 TSourceToken token = list.get( pos ); 043 while ( token.tokentype == ETokenType.ttwhitespace ) 044 { 045 046 if ( token.tokentype == ETokenType.ttwhitespace ) 047 { 048 deleteWhitespaceCount++; 049 } 050 051 if ( opt.opearateSourceToken ) 052 { 053 token.setReplaceToken( SourceTokenConstant.EMPTY ); 054 } 055 056 pos--; 057 if ( pos >= 0 ) 058 { 059 token = token.container.get( pos ); 060 } 061 else 062 { 063 break; 064 } 065 } 066 067 if ( token.tokentype == ETokenType.ttreturn ) 068 { 069 StringBuilder b = new StringBuilder( ); 070 for (int i = 0; i < token.getAstext().length( ); i++ ) 071 { 072 char c = token.getAstext().charAt( i ); 073 if ( c != ' ' ) 074 { 075 b.append( c ); 076 } 077 } 078 token.setAstext(b.toString( )); 079 } 080 081 buf.append( "\nDeleteWhiteSpace:" ).append( deleteWhitespaceCount ); 082 PPLogger.info( 2, buf.toString( ), null ); 083 084 return pos; 085 } 086 087 /** 088 * remove the white space and the return token from the end 089 * 090 * @param list 091 * @param end 092 * @return the pos which is not white space and return token in the source 093 * token list 094 */ 095 public static int removeWhitespaceAndReturnFromEnd( GFmtOpt opt, 096 TSourceTokenList list, int end ) 097 { 098 if ( end == 0 ) 099 { 100 return 0; 101 } 102 103 StringBuilder buf = new StringBuilder( ); 104 buf.append( "removeWhitespaceAndReturnFromEnd:\ntargetToken:" ) 105 .append(list.get(end).getAstext()) 106 .append( " newToken:" ) 107 .append( list.get(end).getAstext().replaceAll( "\\n", "\\\\n" ) ) 108 .append( "(type:" ) 109 .append( list.get( end ).tokentype ) 110 .append( ")" ); 111 int deleteWhitespaceCount = 0; 112 int deleteReturn = 0; 113 114 int pos = end - 1; 115 116 // remove all white space before end item 117 TSourceToken token = list.get( pos ); 118 while ( token.tokentype == ETokenType.ttwhitespace 119 || token.tokentype == ETokenType.ttreturn ) 120 { 121 if ( token.tokentype == ETokenType.ttwhitespace ) 122 { 123 deleteWhitespaceCount++; 124 } 125 126 if ( token.tokentype == ETokenType.ttreturn ) 127 { 128 deleteReturn++; 129 } 130 131 if ( opt.opearateSourceToken ) 132 { 133 token.setReplaceToken( SourceTokenConstant.EMPTY ); 134 } 135 136 pos--; 137 if ( pos >= 0 ) 138 { 139 token = token.container.get( pos ); 140 } 141 else 142 { 143 break; 144 } 145 } 146 147 buf.append( "\nDeleteWhiteSpace:" ) 148 .append( deleteWhitespaceCount ) 149 .append( ", DeleteReturn:" ) 150 .append( deleteReturn ); 151 PPLogger.info( 2, buf.toString( ), null ); 152 153 return pos; 154 } 155 156 /** 157 * remove the white space and the return token from the end 158 * 159 * @param token 160 * @return the pos which is not whitespace and return token in the source 161 * token list 162 */ 163 public static int removeWhitespaceAndReturnFromEnd( GFmtOpt opt, 164 TSourceToken token ) 165 { 166 if ( token == null ) 167 { 168 return -1; 169 } 170 return removeWhitespaceAndReturnFromEnd( opt, 171 token.container, 172 token.posinlist ); 173 } 174 175 /** 176 * remove the white space and the return token before this token and after 177 * this token 178 * 179 * @param opt 180 * @param token 181 */ 182 public static void removeWhitespaceAndReturnFormBeforeAndAfter( 183 GFmtOpt opt, TSourceToken token ) 184 { 185 if ( token == null ) 186 { 187 return; 188 } 189 removeWhitespaceAndReturnFromEnd( opt, token ); 190 removeWhitespaceAndReturnFromStart( opt, 191 token.container, 192 token.posinlist + 1 ); 193 194 } 195 196 /** 197 * remove the white space and the return token 198 * 199 * @param list 200 * @param start 201 * @param end 202 */ 203 public static void removeWhitespaceAndReturn( GFmtOpt opt, 204 TSourceTokenList list, int start, int end ) 205 { 206 StringBuilder buf = new StringBuilder( ); 207 buf.append( "removeWhitespaceAndReturn:\ntargetToken:" ) 208 .append(list.get(end).getAstext()) 209 .append( " newToken:" ) 210 .append( list.get(end).getAstext().replaceAll( "\\n", "\\\\n" ) ) 211 .append( "(type:" ) 212 .append( list.get( end ).tokentype ) 213 .append( ")" ); 214 int deleteWhitespaceCount = 0; 215 int deleteReturn = 0; 216 217 for ( int i = start; i < end; i++ ) 218 { 219 if ( i < 0 || i > list.size( ) ) 220 { 221 return; 222 } 223 TSourceToken token = list.get( i ); 224 if ( token.tokentype == ETokenType.ttwhitespace 225 || token.tokentype == ETokenType.ttreturn ) 226 { 227 228 if ( token.tokentype == ETokenType.ttwhitespace ) 229 { 230 deleteWhitespaceCount++; 231 } 232 233 if ( token.tokentype == ETokenType.ttreturn ) 234 { 235 deleteReturn++; 236 } 237 if ( opt.opearateSourceToken ) 238 { 239 token.setReplaceToken( SourceTokenConstant.EMPTY ); 240 } 241 } 242 } 243 244 buf.append( "\nDeleteWhiteSpace:" ) 245 .append( deleteWhitespaceCount ) 246 .append( ", DeleteReturn:" ) 247 .append( deleteReturn ); 248 PPLogger.info( 2, buf.toString( ), null ); 249 250 } 251 252 /** 253 * remove the white space and the return token from the end 254 * 255 * @param list 256 */ 257 public static int removeWhitespaceAndReturnFromStart( GFmtOpt opt, 258 TSourceTokenList list, int startPos ) 259 { 260 StringBuilder buf = new StringBuilder( ); 261 if ( list.size( ) <= startPos ) 262 { 263 return list.size( ) - 1; 264 } 265 buf.append( "removeWhitespaceAndReturnFromStart:\ntargetToken:" ) 266 .append(list.get(startPos).getAstext()) 267 .append( " newToken:" ) 268 .append( list.get(startPos).getAstext().replaceAll( "\\n", "\\\\n" ) ) 269 .append( "(type:" ) 270 .append( list.get( startPos ).tokentype ) 271 .append( ")" ); 272 int deleteWhitespaceCount = 0; 273 int deleteReturn = 0; 274 275 int pos = startPos; 276 // remove all white space before end item 277 TSourceToken token = list.get( pos ); 278 while ( token.tokentype == ETokenType.ttwhitespace 279 || token.tokentype == ETokenType.ttreturn ) 280 { 281 if ( token.tokentype == ETokenType.ttwhitespace ) 282 { 283 deleteWhitespaceCount++; 284 } 285 286 if ( token.tokentype == ETokenType.ttreturn ) 287 { 288 deleteReturn++; 289 } 290 291 if ( opt.opearateSourceToken ) 292 { 293 token.setReplaceToken( SourceTokenConstant.EMPTY ); 294 } 295 296 pos++; 297 if ( pos < list.size( ) ) 298 { 299 token = token.container.get( pos ); 300 } 301 else 302 { 303 break; 304 } 305 } 306 307 buf.append( "\nDeleteWhiteSpace:" ) 308 .append( deleteWhitespaceCount ) 309 .append( ", DeleteReturn:" ) 310 .append( deleteReturn ); 311 PPLogger.info( 2, buf.toString( ), null ); 312 313 return pos; 314 } 315 316 /** 317 * calculate the text length for all virtual token (which are call before 318 * tokens, after tokens and replace token) 319 * 320 * @param list 321 * @param start 322 * @param end 323 * @return 324 */ 325 public static int textLengthVT( TSourceTokenList list, int start, int end ) 326 { 327 int total = 0; 328 for ( int i = start; i < end; i++ ) 329 { 330 TSourceToken token = list.get( i ); 331 332 if ( token.getTokensBefore( ).size( ) > 0 ) 333 { 334 total += textLengthVT( token.getTokensBefore( ), 335 0, 336 token.getTokensBefore( ).size( ) ); 337 } 338 if ( token.getReplaceToken( ) != null ) 339 { 340 total += ( token.getReplaceToken().getAstext() != null ? token.getReplaceToken().getAstext().length( ) 341 : 0 ); 342 } 343 else 344 { 345 total += ( token.getAstext() != null ? token.getAstext().length( ) : 0 ); 346 } 347 if ( token.getTokensAfter( ).size( ) > 0 ) 348 { 349 total += textLengthVT( token.getTokensAfter( ), 350 0, 351 token.getTokensAfter( ).size( ) ); 352 } 353 } 354 355 return total; 356 } 357 358 /** 359 * calcaulate the text length from a given position to the line start. VT is 360 * the abbr for "virtual token" (which contains before tokens, after tokens 361 * and replace token) 362 * 363 * @param list 364 * @param pos 365 * @return 366 */ 367 public static int curColumnNumberVT( TSourceTokenList list, int pos ) 368 { 369 370 int total = 0; 371 372 for ( int i = pos; i >= 0; i-- ) 373 { 374 TSourceToken token = list.get( i ); 375 376 if(token.tokenstatus == ETokenStatus.tsdeleted) { 377 continue; 378 } 379 380 // the after tokens in the last sourceToken should not calculated. 381 if ( i != pos && token.getTokensAfter( ).size( ) > 0 ) 382 { 383 for ( int j = token.getTokensAfter( ).size( ) - 1; j >= 0; j-- ) 384 { 385 TSourceToken after = token.getTokensAfter( ).get( j ); 386 if ( after.tokentype != ETokenType.ttreturn ) 387 { 388 total += ( after.getAstext() != null ? after.getAstext().length( ) 389 : 0 ); 390 } 391 else 392 { 393 return total; 394 } 395 } 396 } 397 if ( i != pos ) 398 { 399 if ( token.getReplaceToken( ) != null ) 400 { 401 if ( token.getReplaceToken( ).tokentype != ETokenType.ttreturn ) 402 { 403 total += ( token.getReplaceToken().getAstext() != null ? token.getReplaceToken().getAstext().length( ) 404 : 0 ); 405 } 406 else 407 { 408 return total; 409 } 410 } 411 else 412 { 413 if ( token.tokentype != ETokenType.ttreturn ) 414 { 415 if(token.tokentype == ETokenType.ttsimplecomment) { 416 if(token.nextToken()!=null && 417 token.nextToken().getAstext() !=null && 418 token.nextToken().getAstext().trim().length() == 0) { 419 total -= token.nextToken().getAstext().length( ); 420 } 421 continue; 422 } 423 if(isOracleHint(token)) { 424 if(token.nextToken()!=null && 425 token.nextToken().getAstext() !=null && 426 token.nextToken().getAstext().trim().length() == 0) { 427 total -= token.nextToken().getAstext().length( ); 428 } 429 continue; 430 } 431 total += ( token.getAstext() != null ? token.getAstext().length( ) 432 : 0 ); 433 } 434 else 435 { 436 return total; 437 } 438 } 439 } 440 441 if ( token.getTokensBefore( ).size( ) > 0 ) 442 { 443 for ( int j = token.getTokensBefore( ).size( ) - 1; j >= 0; j-- ) 444 { 445 TSourceToken before = token.getTokensBefore( ).get( j ); 446 if ( before.tokentype != ETokenType.ttreturn ) 447 { 448 total += ( before.getAstext() != null ? before.getAstext().length( ) 449 : 0 ); 450 } 451 else 452 { 453 return total; 454 } 455 } 456 } 457 } 458 459 return total; 460 } 461 462 private static boolean isOracleHint(TSourceToken token) { 463 return token!=null && token.getAstext() !=null && token.getAstext().startsWith("/*+"); 464 } 465 466 /** 467 * calcaulate the text length from a given position to the line start. VT is 468 * the abbr for "virtual token" (which contains before tokens, after tokens 469 * and replace token) 470 * 471 * @param token 472 * @return 473 */ 474 public static int curColumnNumberVT( TSourceToken token ) 475 { 476 return curColumnNumberVT( token.container, token.posinlist ); 477 } 478 479 /** 480 * calculate the current indent length VT is the abbr for "virtual token" 481 * (which contains before tokens, after tokens and replace token) 482 * 483 * @param list 484 * @param pos 485 * @return 486 */ 487 public static int curIndentLenVT( TSourceTokenList list, int pos ) 488 { 489 int indentLen = 0; 490 for ( int i = pos; i >= 0; i-- ) 491 { 492 TSourceToken token = list.get( i ); 493 494 if(token.tokenstatus == ETokenStatus.tsdeleted) { 495 continue; 496 } 497 498 // the after tokens in the last sourceToken should not calculated. 499 if ( i != pos && token.getTokensAfter( ).size( ) > 0 ) 500 { 501 for ( int j = token.getTokensAfter( ).size( ) - 1; j >= 0; j-- ) 502 { 503 TSourceToken after = token.getTokensAfter( ).get( j ); 504 if ( after.tokentype == ETokenType.ttreturn ) 505 { 506 return indentLen; 507 } 508 else if ( after.tokentype == ETokenType.ttwhitespace ) 509 { 510 indentLen += ( after.getAstext() != null ? after.getAstext().length( ) 511 : 0 ); 512 } 513 else 514 { 515 indentLen = 0; 516 } 517 } 518 } 519 520 if ( i != pos ) 521 { 522 if ( token.getReplaceToken( ) != null ) 523 { 524 if ( token.getReplaceToken( ).tokentype == ETokenType.ttreturn ) 525 { 526 return indentLen; 527 } 528 else if ( token.getReplaceToken( ).tokentype == ETokenType.ttwhitespace ) 529 { 530 indentLen += ( token.getReplaceToken().getAstext() != null ? token.getReplaceToken().getAstext().length( ) 531 : 0 ); 532 } 533 else 534 { 535 indentLen = 0; 536 } 537 538 } 539 else 540 { 541 if ( token.tokentype == ETokenType.ttreturn ) 542 { 543 return indentLen; 544 } 545 else if ( token.tokentype == ETokenType.ttwhitespace ) 546 { 547 indentLen += ( token.getAstext() != null ? token.getAstext().length( ) 548 : 0 ); 549 } 550 else 551 { 552 indentLen = 0; 553 } 554 } 555 } 556 557 if ( token.getTokensBefore( ).size( ) > 0 ) 558 { 559 for ( int j = token.getTokensBefore( ).size( ) - 1; j >= 0; j-- ) 560 { 561 TSourceToken before = token.getTokensBefore( ).get( j ); 562 if ( before.tokentype == ETokenType.ttreturn ) 563 { 564 return indentLen; 565 } 566 else if ( before.tokentype == ETokenType.ttwhitespace ) 567 { 568 indentLen += ( before.getAstext() != null ? before.getAstext().length( ) 569 : 0 ); 570 } 571 else 572 { 573 indentLen = 0; 574 } 575 } 576 } 577 578 } 579 return indentLen; 580 } 581 582 /** 583 * calculate the current indent length VT is the abbr for "virtual token" 584 * (which contains before tokens, after tokens and replace token) 585 * 586 * @param token 587 * @return 588 */ 589 public static int curIndentLenVT( TSourceToken token ) 590 { 591 return curIndentLenVT( token.container, token.posinlist ); 592 } 593 594 /** 595 * create the white space source token 596 * 597 * @param length 598 * @return 599 */ 600 public static TSourceToken createWhitespaceSourceToken( int length ) 601 { 602 return WhitespaceSourceTokenHelper.createWhitespaceSourceToken( length ); 603 } 604 605 private static TSourceToken noFormatFlag = null; 606 static 607 { 608 noFormatFlag = new TSourceToken( "" ); 609 noFormatFlag.tokentype = ETokenType.ttwhitespace; 610 } 611 612 public static TSourceToken createNoFormatFlagToken( ) 613 { 614 return noFormatFlag; 615 } 616 617 public static TSourceToken createReturnSourceToken( ) 618 { 619 TSourceToken t = new TSourceToken( "\n" ); 620 t.tokentype = ETokenType.ttreturn; 621 return t; 622 } 623 624 private static final TSourceToken statementBreak = createReturnSourceToken( ); 625 626 /** 627 * The one line break placed before a statement that started on the line 628 * of the previous statement's terminator, so that the columns computed 629 * while the statement is formatted start from that break. The empty-line 630 * pass between statements removes it again and lays out the real breaks. 631 */ 632 public static TSourceToken createStatementBreakToken( ) 633 { 634 return statementBreak; 635 } 636 637 /** 638 * combine all the white spaces into a single white space 639 * 640 * @param list 641 * @param startPos 642 * @param endPos 643 */ 644 public static void combineWhitespace( TSourceTokenList list, int startPos, 645 int endPos ) 646 { 647 boolean findWhitespace = false; 648 for ( int i = startPos; i < endPos; i++ ) 649 { 650 if ( i >= 0 && i < list.size( ) ) 651 { 652 TSourceToken token = list.get( i ); 653 654 if(token.tokenstatus == ETokenStatus.tsdeleted) { 655 continue; 656 } 657 658 if ( token.tokentype == ETokenType.ttwhitespace ) 659 { 660 token.setReplaceToken( SourceTokenConstant.EMPTY ); 661 findWhitespace = true; 662 } 663 } 664 } 665 if ( findWhitespace ) 666 { 667 list.get( endPos ) 668 .getTokensBefore( ) 669 .add( createWhitespaceSourceToken( 1 ) ); 670 } 671 } 672 673 /** 674 * insert the source token before the target source token 675 * 676 * @param targetToken 677 * @param needAddToken 678 */ 679 public static void addBefore( GFmtOpt opt, TSourceToken targetToken, 680 TSourceToken needAddToken ) 681 { 682 if ( needAddToken == null ) 683 { 684 return; 685 } 686 687 StringBuilder buf = new StringBuilder( ); 688 buf.append( "AddBefore:\ntargetToken:" ) 689 .append(targetToken.getAstext()) 690 .append( "(pos:" ) 691 .append( targetToken.posinlist ) 692 .append( ")" ) 693 .append( " newToken:" ) 694 .append( needAddToken.getAstext().replaceAll( "\\n", "\\\\n" ) ) 695 .append( "(type:" ) 696 .append( needAddToken.tokentype ) 697 .append( ")" ); 698 699 if ( targetToken.getTokensBefore( ).size( ) > 0 ) 700 { 701 buf.append( "\noldTokenBefore:" ); 702 for ( int i = 0; i < targetToken.getTokensBefore( ).size( ); i++ ) 703 { 704 TSourceToken token = targetToken.getTokensBefore( ).get( i ); 705 706 if(token.tokenstatus == ETokenStatus.tsdeleted) { 707 continue; 708 } 709 710 buf.append( "[" ) 711 .append( token.getAstext() != null ? token.getAstext().replaceAll( "\\n", 712 "\\\\n" ) 713 : "" ) 714 .append( "] " ) 715 .append( "(type:" ) 716 .append( token.tokentype ) 717 .append( ")" ); 718 } 719 } 720 721 PPLogger.info( 2, buf.toString( ), null ); 722 723 if ( opt.opearateSourceToken ) 724 { 725 targetToken.getTokensBefore( ).add( needAddToken ); 726 } 727 } 728 729 public static void addAfter( GFmtOpt opt, TSourceToken targetToken, 730 TSourceToken needAddToken ) 731 { 732 if ( needAddToken == null ) 733 { 734 return; 735 } 736 737 StringBuilder buf = new StringBuilder( ); 738 buf.append( "AddAfter:\ntargetToken:" ) 739 .append(targetToken.getAstext()) 740 .append( "(pos:" ) 741 .append( targetToken.posinlist ) 742 .append( ")" ) 743 .append( " newToken:" ) 744 .append( needAddToken.getAstext() != null ? needAddToken.getAstext().replaceAll( "\\n", 745 "\\\\n" ) 746 : "" ) 747 .append( "(type:" ) 748 .append( needAddToken.tokentype ) 749 .append( ")" ); 750 751 if ( targetToken.getTokensAfter( ).size( ) > 0 ) 752 { 753 buf.append( "\noldTokenAfter:" ); 754 for ( int i = 0; i < targetToken.getTokensAfter( ).size( ); i++ ) 755 { 756 TSourceToken token = targetToken.getTokensAfter( ).get( i ); 757 758 if(token.tokenstatus == ETokenStatus.tsdeleted) { 759 continue; 760 } 761 762 buf.append( "[" ) 763 .append( token.getAstext() != null ? token.getAstext().replaceAll( "\\n", 764 "\\\\n" ) 765 : "" ) 766 .append( "] " ) 767 .append( "(type:" ) 768 .append( token.tokentype ) 769 .append( ")" ); 770 } 771 } 772 773 PPLogger.info( 2, buf.toString( ), null ); 774 775 if ( opt.opearateSourceToken ) 776 { 777 targetToken.getTokensAfter( ).add( needAddToken ); 778 } 779 } 780 781 /** 782 * remove all source token in the before source token list VT is the abbr 783 * for "virtual token" (that this method will operates before tokens, after 784 * tokens and replace token) 785 * 786 * @param targetToken 787 */ 788 public static void removeAllBeforeTokenVT( GFmtOpt opt, 789 TSourceToken targetToken ) 790 { 791 StringBuilder buf = new StringBuilder( ); 792 buf.append( "RemoveAllBeforeTokenVT:" ); 793 while ( targetToken.getTokensBefore( ).size( ) > 0 ) 794 { 795 TSourceToken token = targetToken.getTokensBefore( ).get( 0 ); 796 797 if(token.tokenstatus == ETokenStatus.tsdeleted) { 798 continue; 799 } 800 801 buf.append( "\nremoveToken:" ) 802 .append( "[" ) 803 .append( token.getAstext() != null ? token.getAstext().replaceAll( "\\n", 804 "\\\\n" ) 805 : "" ) 806 .append( "] " ) 807 .append( "(type:" ) 808 .append( token.tokentype ) 809 .append( ")" ); 810 if ( opt.opearateSourceToken ) 811 { 812 targetToken.getTokensBefore( ).remove( 0 ); 813 } 814 } 815 PPLogger.info( 2, buf.toString( ), null ); 816 817 } 818 819 public static void removeThisToken( TSourceToken token ) 820 { 821 token.setReplaceToken( SourceTokenOperator.createWhitespaceSourceToken( 1 ) ); 822 } 823}