001package gudusoft.gsqlparser.parser; 002 003import gudusoft.gsqlparser.EDbVendor; 004import gudusoft.gsqlparser.TBaseType; 005import gudusoft.gsqlparser.TCustomLexer; 006import gudusoft.gsqlparser.TCustomParser; 007import gudusoft.gsqlparser.TCustomSqlStatement; 008import gudusoft.gsqlparser.TLexerClickhouse; 009import gudusoft.gsqlparser.TParserClickhouse; 010import gudusoft.gsqlparser.TGSqlParser; 011import gudusoft.gsqlparser.TSourceToken; 012import gudusoft.gsqlparser.TSourceTokenList; 013import gudusoft.gsqlparser.TStatementList; 014import gudusoft.gsqlparser.TSyntaxError; 015import gudusoft.gsqlparser.EFindSqlStateType; 016import gudusoft.gsqlparser.ETokenType; 017import gudusoft.gsqlparser.ETokenStatus; 018import gudusoft.gsqlparser.ESqlStatementType; 019import gudusoft.gsqlparser.EErrorType; 020import gudusoft.gsqlparser.stmt.TUnknownSqlStatement; 021import gudusoft.gsqlparser.sqlcmds.ISqlCmds; 022import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory; 023import gudusoft.gsqlparser.compiler.TContext; 024import gudusoft.gsqlparser.sqlenv.TSQLEnv; 025import gudusoft.gsqlparser.compiler.TGlobalScope; 026import gudusoft.gsqlparser.compiler.TFrame; 027 028import java.util.ArrayList; 029import java.util.List; 030import java.util.Stack; 031 032/** 033 * ClickHouse database SQL parser implementation. 034 * 035 * <p>This parser handles ClickHouse-specific SQL syntax including: 036 * <ul> 037 * <li>Standard SQL DML/DDL (SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, ALTER)</li> 038 * <li>ClickHouse-specific ENGINE clause in CREATE TABLE</li> 039 * <li>PREWHERE, FINAL, SAMPLE, ARRAY JOIN clauses</li> 040 * <li>FORMAT and SETTINGS clauses</li> 041 * <li>ClickHouse data types (Array, Tuple, Map, Nullable, LowCardinality, etc.)</li> 042 * </ul> 043 * 044 * <p><b>Design Notes:</b> 045 * <ul> 046 * <li>Extends {@link AbstractSqlParser}</li> 047 * <li>Based on MySQL grammar (ClickHouse shares backtick quoting, # comments, ENGINE clause)</li> 048 * <li>No stored procedure support (ClickHouse has no stored procedures)</li> 049 * <li>Delimiter character: ';' for SQL statements</li> 050 * </ul> 051 * 052 * @see AbstractSqlParser 053 * @see TLexerClickhouse 054 * @see TParserClickhouse 055 * @since 3.2.0.0 056 */ 057public class ClickhouseSqlParser extends AbstractSqlParser { 058 059 // ========== Lexer and Parser Instances ========== 060 061 /** The ClickHouse lexer used for tokenization (public for TGSqlParser.getFlexer()) */ 062 public TLexerClickhouse flexer; 063 private TParserClickhouse fparser; 064 065 /** 066 * PARALLEL and WITH of each {@code stmt1 PARALLEL WITH stmt2}, as 067 * clickhousePostTokenAdjust leaves them: the splitter ends stmt1 at them 068 * and gives them to neither statement. 069 */ 070 private final java.util.Set<TSourceToken> parallelWithSeparators = 071 java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<TSourceToken, Boolean>()); 072 073 /** 074 * Tokens of INSERT ... FORMAT rows whose text went into the first token of their rows; the pre-pass 075 * drops them all at its end, so a script of many inserts is not re-shifted once per insert. 076 */ 077 private final java.util.Set<TSourceToken> foldedRowTokens = 078 java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<TSourceToken, Boolean>()); 079 080 // ========== Constructor ========== 081 082 /** 083 * Construct ClickHouse SQL parser. 084 * <p> 085 * Configures the parser for ClickHouse database with default delimiter: semicolon (;) 086 */ 087 public ClickhouseSqlParser() { 088 super(EDbVendor.dbvclickhouse); 089 090 // Set delimiter character - ClickHouse uses semicolon 091 this.delimiterChar = ';'; 092 this.defaultDelimiterStr = ";"; 093 094 // Create lexer once - will be reused for all parsing operations 095 this.flexer = new TLexerClickhouse(); 096 this.flexer.delimiterchar = this.delimiterChar; 097 this.flexer.defaultDelimiterStr = this.defaultDelimiterStr; 098 099 // CRITICAL: Set lexer for inherited getanewsourcetoken() method 100 this.lexer = this.flexer; 101 102 // Create parser once - will be reused for all parsing operations 103 this.fparser = new TParserClickhouse(null); 104 this.fparser.lexer = this.flexer; 105 } 106 107 // ========== AbstractSqlParser Abstract Methods Implementation ========== 108 109 @Override 110 protected TCustomLexer getLexer(ParserContext context) { 111 return this.flexer; 112 } 113 114 @Override 115 protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) { 116 this.fparser.sourcetokenlist = tokens; 117 return this.fparser; 118 } 119 120 @Override 121 protected void tokenizeVendorSql() { 122 doclickhousetexttotokenlist(); 123 } 124 125 @Override 126 protected void setupVendorParsersForExtraction() { 127 this.fparser.sqlcmds = this.sqlcmds; 128 this.fparser.sourcetokenlist = this.sourcetokenlist; 129 } 130 131 @Override 132 protected void extractVendorRawStatements(SqlParseResult.Builder builder) { 133 doclickhousegetrawsqlstatements(builder); 134 } 135 136 @Override 137 protected TStatementList performParsing(ParserContext context, 138 TCustomParser parser, 139 TCustomParser secondaryParser, 140 TSourceTokenList tokens, 141 TStatementList rawStatements) { 142 this.sourcetokenlist = tokens; 143 this.parserContext = context; 144 this.sqlstatements = rawStatements; 145 146 // Initialize sqlcmds for the parser 147 this.sqlcmds = SqlCmdsFactory.get(vendor); 148 this.fparser.sqlcmds = this.sqlcmds; 149 150 // Initialize global context for statement parsing 151 initializeGlobalContext(); 152 153 // Parse each statement 154 for (int i = 0; i < sqlstatements.size(); i++) { 155 TCustomSqlStatement stmt = sqlstatements.getRawSql(i); 156 157 try { 158 stmt.setFrameStack(frameStack); 159 int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree()); 160 161 // Handle error recovery for CREATE TABLE statements if enabled 162 boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE; 163 if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) { 164 handleCreateTableErrorRecovery(stmt); 165 } 166 167 if ((parseResult != 0) || (stmt.getErrorCount() > 0)) { 168 copyErrorsFromStatement(stmt); 169 } 170 } catch (Exception ex) { 171 handleStatementParsingException(stmt, i, ex); 172 continue; 173 } 174 } 175 176 // Clean up frame stack 177 if (globalFrame != null) { 178 globalFrame.popMeFromStack(frameStack); 179 } 180 181 return this.sqlstatements; 182 } 183 184 /** 185 * Handle error recovery for CREATE TABLE statements. 186 */ 187 private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) { 188 if ((stmt.sqlstatementtype != ESqlStatementType.sstcreatetable) || TBaseType.c_createTableStrictParsing) { 189 return; 190 } 191 192 int nested = 0; 193 boolean isIgnore = false, isFoundIgnoreToken = false; 194 TSourceToken firstIgnoreToken = null; 195 196 for (int k = 0; k < stmt.sourcetokenlist.size(); k++) { 197 TSourceToken st = stmt.sourcetokenlist.get(k); 198 if (isIgnore) { 199 if (st.issolidtoken() && (st.tokencode != ';')) { 200 isFoundIgnoreToken = true; 201 if (firstIgnoreToken == null) { 202 firstIgnoreToken = st; 203 } 204 } 205 if (st.tokencode != ';') { 206 st.tokencode = TBaseType.sqlpluscmd; 207 } 208 continue; 209 } 210 if (st.tokencode == (int) ')') { 211 nested--; 212 if (nested == 0) { 213 boolean isSelect = false; 214 TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1); 215 if (st1 != null) { 216 TSourceToken st2 = st.searchToken((int) '(', 2); 217 if (st2 != null) { 218 TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3); 219 isSelect = (st3 != null); 220 } 221 } 222 if (!isSelect) isIgnore = true; 223 } 224 } else if (st.tokencode == (int) '(') { 225 nested++; 226 } 227 } 228 229 if (isFoundIgnoreToken) { 230 stmt.clearError(); 231 stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree()); 232 } 233 } 234 235 // ========== ClickHouse-Specific Tokenization ========== 236 237 /** 238 * Perform ClickHouse-specific tokenization. 239 * <p> 240 * Based on MySQL tokenization but simplified - no DELIMITER command, 241 * no stored procedure handling. Handles ClickHouse-specific token 242 * transformations including :: typecast operator splitting. 243 */ 244 private void doclickhousetexttotokenlist() { 245 TSourceToken asourcetoken; 246 int yychar; 247 248 asourcetoken = getanewsourcetoken(); 249 if (asourcetoken == null) return; 250 yychar = asourcetoken.tokencode; 251 252 while (yychar > 0) { 253 // Handle :: typecast operator: the lexer produces mysqllabel (identifier:) 254 // followed by bind_v (:name) for "identifier::TypeName". We need to 255 // rewrite as CAST(identifier AS TypeName). 256 if (yychar == TBaseType.mysqllabel) { 257 // Append BEFORE fetching the lookahead: getanewsourcetoken's 258 // whitespace merge reads sourcetokenlist.get(curpos), which is 259 // out of bounds while this token is held back — "SELECT {a: b}" 260 // (label then whitespace) crashed the whole tokenize and the 261 // parser silently returned zero tokens. The bind_v rewrite 262 // removes the token again below. 263 sourcetokenlist.add(asourcetoken); 264 TSourceToken nextToken = getanewsourcetoken(); 265 if (nextToken != null && nextToken.tokencode == TBaseType.bind_v) { 266 sourcetokenlist.remove(sourcetokenlist.size() - 1); 267 // We have mysqllabel + bind_v pattern = "ident:" + ":TypeName" 268 // Rewrite as: CAST ( identifier AS TypeName ) 269 String labelText = asourcetoken.toString(); 270 String identText = labelText.substring(0, labelText.length() - 1); 271 String bindText = nextToken.toString(); 272 String typeText = bindText.substring(1); 273 int line = (int) asourcetoken.lineNo; 274 int col = (int) asourcetoken.columnNo; 275 276 // Check if preceding token is '.' (qualified name like t1.col::Type) 277 // If so, pull qualified name prefix into the CAST expression 278 List<TSourceToken> qualifiedPrefix = new ArrayList<>(); 279 int lastIdx = sourcetokenlist.size() - 1; 280 // Skip trailing whitespace 281 while (lastIdx >= 0 && (sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttwhitespace 282 || sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttreturn)) { 283 lastIdx--; 284 } 285 if (lastIdx >= 0 && sourcetokenlist.get(lastIdx).tokentype == ETokenType.ttperiod) { 286 // Walk back collecting name.name.name. pattern 287 int prefixEnd = lastIdx; 288 int idx = lastIdx; 289 while (idx >= 0) { 290 TSourceToken t = sourcetokenlist.get(idx); 291 if (t.tokentype == ETokenType.ttperiod) { 292 idx--; 293 } else if (t.tokentype == ETokenType.ttidentifier || t.tokentype == ETokenType.ttkeyword) { 294 idx--; 295 // Skip whitespace between identifier and dot 296 while (idx >= 0 && (sourcetokenlist.get(idx).tokentype == ETokenType.ttwhitespace 297 || sourcetokenlist.get(idx).tokentype == ETokenType.ttreturn)) { 298 idx--; 299 } 300 // Check if next non-ws token is another dot (continue) or stop 301 if (idx >= 0 && sourcetokenlist.get(idx).tokentype == ETokenType.ttperiod) { 302 continue; 303 } else { 304 break; 305 } 306 } else { 307 break; 308 } 309 } 310 int prefixStart = idx + 1; 311 // Collect prefix tokens and remove from sourcetokenlist 312 for (int i = prefixStart; i <= prefixEnd; i++) { 313 qualifiedPrefix.add(sourcetokenlist.get(i)); 314 } 315 // Remove prefix tokens from end of sourcetokenlist 316 while (sourcetokenlist.size() > prefixStart) { 317 sourcetokenlist.remove(sourcetokenlist.size() - 1); 318 } 319 } 320 321 // Emit: CAST ( 322 sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col)); 323 sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col)); 324 325 // Re-emit qualified prefix if any (e.g., "t1.") 326 for (TSourceToken prefixToken : qualifiedPrefix) { 327 prefixToken.posinlist = sourcetokenlist.size(); 328 sourcetokenlist.add(prefixToken); 329 } 330 331 // Emit: identifier or integer 332 asourcetoken.setAstext(identText); 333 if (identText.matches("\\d+")) { 334 asourcetoken.tokencode = TBaseType.iconst; 335 asourcetoken.tokentype = ETokenType.ttnumber; 336 } else { 337 asourcetoken.tokencode = TBaseType.ident; 338 asourcetoken.tokentype = ETokenType.ttidentifier; 339 } 340 sourcetokenlist.add(asourcetoken); 341 342 // Emit: AS 343 sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col)); 344 345 // Emit: TypeName 346 nextToken.setAstext(typeText); 347 nextToken.tokencode = TBaseType.ident; 348 nextToken.tokentype = ETokenType.ttidentifier; 349 sourcetokenlist.add(nextToken); 350 351 // Check for parameterized types: TypeName(params) 352 emitTypeParamsAndClose(line, col); 353 354 asourcetoken = getanewsourcetoken(); 355 if (asourcetoken == null) break; 356 yychar = asourcetoken.tokencode; 357 continue; 358 } else { 359 // Not followed by bind_v; the mysqllabel is already in the 360 // list, continue with the lookahead token. 361 if (nextToken == null) break; 362 asourcetoken = nextToken; 363 yychar = asourcetoken.tokencode; 364 continue; 365 } 366 } 367 368 // Handle standalone :: typecast token (e.g., after string literals: '2024-01-15'::Date) 369 if (yychar == TBaseType.typecast) { 370 rewriteStandaloneTypecast(asourcetoken); 371 asourcetoken = getanewsourcetoken(); 372 if (asourcetoken == null) break; 373 yychar = asourcetoken.tokencode; 374 continue; 375 } 376 377 if (yychar == TBaseType.bind_v) { 378 // ClickHouse has no :name bind variables; a bind_v that 379 // survived the :: rewrite above is a map-literal colon glued 380 // to its value ({'key1':1}). Split into ':' plus the value. 381 String bindText = asourcetoken.toString(); 382 int bLine = (int) asourcetoken.lineNo; 383 int bCol = (int) asourcetoken.columnNo; 384 sourcetokenlist.add(createSyntheticToken(":", ':', ETokenType.ttcolon, bLine, bCol)); 385 String valText = bindText.substring(1); 386 boolean allDigits = valText.length() > 0; 387 for (int ci = 0; ci < valText.length(); ci++) { 388 if (!Character.isDigit(valText.charAt(ci))) { allDigits = false; break; } 389 } 390 if (allDigits) { 391 sourcetokenlist.add(createSyntheticToken(valText, TBaseType.iconst, ETokenType.ttnumber, bLine, bCol + 1)); 392 } else { 393 sourcetokenlist.add(createSyntheticToken(valText, TBaseType.ident, ETokenType.ttidentifier, bLine, bCol + 1)); 394 } 395 asourcetoken = getanewsourcetoken(); 396 if (asourcetoken == null) break; 397 yychar = asourcetoken.tokencode; 398 continue; 399 } 400 401 // Tuple element access a.2: the lexer greedily reads ".2" as a 402 // float constant. When a float starting with '.' directly follows 403 // an identifier or a closing bracket, split it into a dedicated 404 // tuple-access dot token plus an integer constant. The dot gets 405 // its own token (CH_TUPLE_DOT) rather than '.' so the grammar 406 // rule cannot collide with qualified names like left.col. 407 if ((yychar == TBaseType.fconst) && asourcetoken.toString().startsWith(".") 408 && asourcetoken.toString().indexOf('.', 1) < 0) { 409 TSourceToken lastSolid = null; 410 for (int li = sourcetokenlist.size() - 1; li >= 0; li--) { 411 if (sourcetokenlist.get(li).issolidtoken()) { lastSolid = sourcetokenlist.get(li); break; } 412 } 413 if ((lastSolid != null) 414 && ((lastSolid.tokentype == ETokenType.ttidentifier) 415 || (lastSolid.tokencode == ')') || (lastSolid.tokencode == ']'))) { 416 int line = (int) asourcetoken.lineNo; 417 int col = (int) asourcetoken.columnNo; 418 sourcetokenlist.add(createSyntheticToken(".", TBaseType.rrw_clickhouse_tuple_dot, ETokenType.ttperiod, line, col)); 419 TSourceToken num = createSyntheticToken(asourcetoken.toString().substring(1), TBaseType.iconst, ETokenType.ttnumber, line, col + 1); 420 sourcetokenlist.add(num); 421 asourcetoken = getanewsourcetoken(); 422 if (asourcetoken == null) break; 423 yychar = asourcetoken.tokencode; 424 continue; 425 } 426 } 427 428 sourcetokenlist.add(asourcetoken); 429 430 asourcetoken = getanewsourcetoken(); 431 if (asourcetoken == null) break; 432 433 yychar = asourcetoken.tokencode; 434 } 435 436 clickhousePostTokenAdjust(); 437 } 438 439 /** 440 * Post-tokenize adjustments that need lookahead/lookbehind over the whole 441 * list. Three rewrites, all ClickHouse-specific: 442 * 443 * 1. IN whose next solid token is PARTITION becomes RW_IN_BEFORE_PARTITION. 444 * ALTER TABLE t UPDATE a = 1 IN PARTITION p WHERE ... is LALR(1)-blocked 445 * with plain IN: "a = 1 IN (...)" is a legal value expression, so the 446 * parser must shift IN into the expression and then dies at PARTITION. 447 * "x IN PARTITION" is never a legal expression, so the rewrite is safe 448 * globally; the grammar accepts both spellings via ch_in_before_partition. 449 * 450 * 2. FROM/TO demoted to identifiers where the keyword reading is impossible: 451 * after '.' (g.from), or when the next solid token is ',', ';' or ')' 452 * (SELECT from, to, label ... ORDER BY from). ClickHouse itself accepts 453 * these as column names (WITH RECURSIVE documentation examples). 454 * 455 * 3. INSERT INTO t FORMAT <name> followed by inline data rows: everything 456 * after the format name up to the statement's ';' is raw data in the 457 * named format, not SQL. It becomes one sqlpluscmd token holding the 458 * text as written, which TCustomParser already skips (same mechanism as 459 * handleCreateTableErrorRecovery above). Skipped when a SELECT comes 460 * before FORMAT: that FORMAT belongs to the query. 461 */ 462 private void clickhousePostTokenAdjust() { 463 parallelWithSeparators.clear(); 464 foldedRowTokens.clear(); 465 // Token-list surgery above (the :: CAST rewrite and the bind/float 466 // splits) inserts or removes tokens, but the lexer stamps posinlist 467 // from its own running count, so every token after the first edit 468 // carries a stale position. The statement splitter walks tokens BY 469 // posinlist (nextsolidtoken), so a stale value makes it read the 470 // wrong token and mistype the NEXT statement as sstinvalid — this 471 // predates the splits: SELECT a::UInt32; CREATE TABLE ... already 472 // mistyped the CREATE. Renumber once so positions match reality. 473 for (int ri = 0; ri < sourcetokenlist.size(); ri++) { 474 sourcetokenlist.get(ri).posinlist = ri; 475 } 476 477 TSourceToken prevSolid = null; 478 for (int i = 0; i < sourcetokenlist.size(); i++) { 479 TSourceToken st = sourcetokenlist.get(i); 480 if (!st.issolidtoken()) continue; 481 482 if (st.tokencode == TBaseType.rrw_clickhouse_in) { 483 TSourceToken nx = nextSolidToken(i); 484 if ((nx != null) && (nx.tokencode == TBaseType.rrw_clickhouse_partition)) { 485 st.tokencode = TBaseType.rrw_clickhouse_in_before_partition; 486 } 487 } else if ((st.tokencode == TBaseType.rrw_from) || (st.tokencode == TBaseType.rrw_to)) { 488 TSourceToken nx = nextSolidToken(i); 489 boolean afterDot = (prevSolid != null) && (prevSolid.tokencode == '.'); 490 boolean beforeCloser = (nx != null) 491 && ((nx.tokencode == ',') || (nx.tokencode == ';') || (nx.tokencode == ')')); 492 if (afterDot || beforeCloser) { 493 st.tokencode = TBaseType.ident; 494 st.tokentype = ETokenType.ttidentifier; 495 } 496 } else if (st.tokencode == TBaseType.rrw_clickhouse_at) { 497 // AT is only a keyword in "AT TIME ZONE" / "AT LOCAL"; anywhere 498 // else it stays a plain identifier so aliases and columns named 499 // "at" keep parsing exactly as before AT was tokenized. 500 TSourceToken nx = nextSolidToken(i); 501 if ((nx == null) || ((nx.tokencode != TBaseType.rrw_clickhouse_time) 502 && (nx.tokencode != TBaseType.rrw_clickhouse_local))) { 503 st.tokencode = TBaseType.ident; 504 st.tokentype = ETokenType.ttidentifier; 505 } 506 } else if (st.tokencode == TBaseType.rrw_clickhouse_zone) { 507 // ZONE is only a keyword directly after TIME. 508 if ((prevSolid == null) || (prevSolid.tokencode != TBaseType.rrw_clickhouse_time)) { 509 st.tokencode = TBaseType.ident; 510 st.tokentype = ETokenType.ttidentifier; 511 } 512 } else if (st.tokencode == '.') { 513 TSourceToken nx = nextSolidToken(i); 514 boolean prevOk = (prevSolid != null) 515 && ((prevSolid.tokentype == ETokenType.ttidentifier) 516 || (prevSolid.tokencode == ')') || (prevSolid.tokencode == ']')); 517 if (prevOk && (nx != null) && (nx.tokencode == TBaseType.iconst)) { 518 st.tokencode = TBaseType.rrw_clickhouse_tuple_dot; 519 } 520 } else if (st.tokencode == TBaseType.rrw_insert) { 521 rewriteInsertFormatRawData(i); 522 } else if ((st.tokencode == TBaseType.ident) && st.toString().equalsIgnoreCase("PARALLEL")) { 523 // stmt1 PARALLEL WITH stmt2: an execution combinator GSP does not 524 // model. PARALLEL and WITH become a separator between the two 525 // statements, which both parse with full ASTs; the concurrency 526 // annotation itself is not modelled. Guarded on the token after 527 // WITH being a statement-starting keyword, so ORDER BY parallel 528 // WITH FILL, LIMIT parallel WITH TIES and GROUP BY parallel WITH 529 // TOTALS stay untouched. 530 TSourceToken w = nextSolidToken(i); 531 if ((w != null) && (w.tokencode == TBaseType.rrw_with)) { 532 TSourceToken head = nextSolidToken(w.posinlist); 533 if ((head != null) && isClickhouseStatementStart(head.tokencode)) { 534 // The parser skips sqlpluscmd, and the splitter ends stmt1 535 // at the pair and gives it to neither statement, so each 536 // statement's text ("DROP TABLE t1", "DROP TABLE t2;") 537 // re-parses on its own. The words stay keywords with their 538 // text, so a formatter prints and cases them and a content 539 // check compares them; written as ';' they came out of 540 // every formatter as "; ;". 541 st.tokencode = TBaseType.sqlpluscmd; 542 st.tokentype = ETokenType.ttkeyword; 543 w.tokencode = TBaseType.sqlpluscmd; 544 w.tokentype = ETokenType.ttkeyword; 545 parallelWithSeparators.add(st); 546 parallelWithSeparators.add(w); 547 } 548 } 549 } 550 551 prevSolid = st; 552 } 553 dropFoldedRowTokens(); 554 } 555 556 private boolean isClickhouseStatementStart(int tokencode) { 557 switch (tokencode) { 558 case TBaseType.rrw_select: 559 case TBaseType.rrw_insert: 560 case TBaseType.rrw_delete: 561 case TBaseType.rrw_create: 562 case TBaseType.rrw_drop: 563 case TBaseType.rrw_truncate: 564 case 304: // UPDATE 565 case 426: // ALTER 566 case 476: // RENAME 567 case 504: // OPTIMIZE 568 case 1181: // DETACH 569 case 1182: // ATTACH 570 return true; 571 default: 572 return false; 573 } 574 } 575 576 private TSourceToken nextSolidToken(int from) { 577 for (int j = from + 1; j < sourcetokenlist.size(); j++) { 578 TSourceToken t = sourcetokenlist.get(j); 579 if (t.issolidtoken()) return t; 580 } 581 return null; 582 } 583 584 private void rewriteInsertFormatRawData(int insertPos) { 585 // The insert's FORMAT clause is "FORMAT <name>" after the target, outside parentheses, so a 586 // column, table, database or setting named format is not it: "(a, format)", "INTO db.format", 587 // "SETTINGS format = 1". VALUES, SELECT or WITH start the data or the query instead; a FORMAT 588 // after them is the query's. 589 int formatPos = -1; 590 int depth = 0; 591 TSourceToken prev = null; 592 for (int j = afterInsertTarget(insertPos); j < sourcetokenlist.size(); j++) { 593 TSourceToken t = sourcetokenlist.get(j); 594 if (!t.issolidtoken()) continue; 595 if (t.tokencode == ';') return; 596 if (t.tokencode == TBaseType.rrw_select) return; 597 if (t.tokencode == '(') { 598 depth++; 599 } else if (t.tokencode == ')') { 600 if (depth > 0) depth--; 601 } else if ((depth == 0) && !isName(prev, j)) { 602 if ((t.tokencode == TBaseType.rrw_values) || (t.tokencode == TBaseType.rrw_with)) return; 603 if (t.tokencode == TBaseType.rrw_clickhouse_format) { 604 TSourceToken name = nextSolidToken(j); 605 if ((name != null) && !isPunctuation(name)) { 606 formatPos = j; 607 break; 608 } 609 } 610 } 611 prev = t; 612 } 613 if (formatPos < 0) return; 614 int namePos = nextSolidToken(formatPos).posinlist; 615 // Everything after "FORMAT <name>" is data to ClickHouse, a row reading "select" or only a 616 // comment or a blank line included, so the end is found before looking at what the rows hold 617 int end = sourcetokenlist.size(); 618 for (int j = namePos + 1; j < sourcetokenlist.size(); j++) { 619 TSourceToken t = sourcetokenlist.get(j); 620 if (t.tokencode == ';') { 621 end = j; 622 break; 623 } 624 if (t.tokencode == TBaseType.cmtdoublehyphen) { 625 // "--" or "#" in the rows is data, not a comment: a ';' in it 626 // still ends the rows ("1,--none--;"), as it did when "--x" 627 // lexed as two minus signs 628 int semi = t.toString().indexOf(';'); 629 if (semi > 0) { 630 splitRawRowsAt(j, semi); 631 end = j + 1; 632 break; 633 } 634 } 635 } 636 // ClickHouse reads the data from after the format name, past spaces, tabs and form feeds and 637 // then one line break. Those blanks stay whitespace; the rest is one token holding the rows as 638 // written, line breaks, tabs and case included, so a formatter cannot join or re-case them 639 // ("1,a\n2,b" came out as the single row "1,a 2,b") 640 int start = namePos + 1; 641 while ((start < end) && isBlankBeforeRows(sourcetokenlist.get(start))) { 642 start++; 643 } 644 if (start >= end) return; // no rows: "FORMAT CSV;" 645 TSourceToken rows = sourcetokenlist.get(start); 646 StringBuilder text = new StringBuilder(); 647 for (int k = start; k < end; k++) { 648 text.append(sourcetokenlist.get(k).toString()); 649 } 650 rows.setString(text.toString()); 651 rows.tokencode = TBaseType.sqlpluscmd; 652 rows.tokentype = ETokenType.ttsqlpluscmd; 653 for (int k = start + 1; k < end; k++) { 654 // dropped in one pass at the end of the pre-pass; its rules skip sqlpluscmd until then 655 TSourceToken t = sourcetokenlist.get(k); 656 t.tokencode = TBaseType.sqlpluscmd; 657 foldedRowTokens.add(t); 658 } 659 } 660 661 /** Drop the tokens folded into their rows' token, in one pass, and renumber the list. */ 662 private void dropFoldedRowTokens() { 663 if (foldedRowTokens.isEmpty()) return; 664 int kept = 0; 665 for (int k = 0; k < sourcetokenlist.size(); k++) { 666 TSourceToken t = sourcetokenlist.get(k); 667 if (foldedRowTokens.contains(t)) continue; 668 t.posinlist = kept; 669 sourcetokenlist.set(kept++, t); 670 } 671 for (int k = sourcetokenlist.size() - 1; k >= kept; k--) { 672 sourcetokenlist.remove(k); 673 } 674 foldedRowTokens.clear(); 675 } 676 677 /** 678 * The index after the target of the INSERT at {@code insertPos}: "INTO [TABLE | FUNCTION] name", the 679 * name's parts joined by '.', so a target named format, values, table or into ("INTO db.table FORMAT 680 * CSV") is never read as a keyword. The index after INSERT when no INTO follows. 681 */ 682 private int afterInsertTarget(int insertPos) { 683 TSourceToken into = nextSolidToken(insertPos); 684 if ((into == null) || (into.tokencode != TBaseType.rrw_into)) return insertPos + 1; 685 TSourceToken part = nextSolidToken(into.posinlist); 686 if (part == null) return into.posinlist + 1; 687 if ((part.tokencode == TBaseType.rrw_table) || (part.tokencode == TBaseType.rrw_function)) { 688 TSourceToken after = nextSolidToken(part.posinlist); 689 // the keyword, unless it is itself a database name ("INTO table.t") 690 if ((after != null) && (after.tokencode != '.')) part = after; 691 } 692 if (isPunctuation(part)) return part.posinlist; 693 for (TSourceToken dot = nextSolidToken(part.posinlist); (dot != null) && (dot.tokencode == '.'); 694 dot = nextSolidToken(part.posinlist)) { 695 TSourceToken next = nextSolidToken(dot.posinlist); 696 if (next == null) break; 697 part = next; 698 } 699 return part.posinlist + 1; 700 } 701 702 /** 703 * Whether the word at {@code pos}, after the solid token {@code prev}, is used as a name: a part of a 704 * dotted name, a setting ("format = 1") or a list item, not the keyword it is spelled like. 705 */ 706 private boolean isName(TSourceToken prev, int pos) { 707 if ((prev != null) && (prev.tokencode == '.')) return true; 708 TSourceToken next = nextSolidToken(pos); 709 return (next != null) && ((next.tokencode == '.') || (next.tokencode == '=') || (next.tokencode == ',') 710 || (next.tokencode == ')')); 711 } 712 713 private static boolean isPunctuation(TSourceToken t) { 714 switch (t.tokentype) { 715 case ttsemicolon: 716 case ttcomma: 717 case ttperiod: 718 case ttequals: 719 case ttleftparenthesis: 720 case ttrightparenthesis: 721 return true; 722 default: 723 return false; 724 } 725 } 726 727 /** A whitespace token made only of the blanks ClickHouse skips before the first line break. */ 728 private static boolean isBlankBeforeRows(TSourceToken t) { 729 if (t.tokentype != ETokenType.ttwhitespace) return false; 730 String s = t.toString(); 731 for (int i = 0; i < s.length(); i++) { 732 char c = s.charAt(i); 733 if ((c != ' ') && (c != '\t') && (c != '\f')) return false; 734 } 735 return true; 736 } 737 738 /** 739 * Split the raw-row token at {@code j} at the ';' at index {@code semi} of its text: the text 740 * before stays in the rows, the ';' ends them, and the rest of the line is SQL again, lexed as 741 * such. 742 */ 743 private void splitRawRowsAt(int j, int semi) { 744 TSourceToken t = sourcetokenlist.get(j); 745 String text = t.toString(); 746 String rest = text.substring(semi + 1); 747 t.setString(text.substring(0, semi)); 748 // no longer a comment that runs to the end of the line: a formatter 749 // prints it in place, before its ';', and does not re-case it 750 t.tokentype = ETokenType.ttsqlpluscmd; 751 TSourceToken semicolon = createSyntheticToken(";", ';', ETokenType.ttsemicolon, (int) t.lineNo, 752 (int) t.columnNo + semi); 753 semicolon.offset = t.offset + semi; 754 semicolon.setDbvendor(t.getDbvendor()); 755 int at = j + 1; 756 sourcetokenlist.add(at++, semicolon); 757 if (!rest.isEmpty()) { 758 TGSqlParser sub = new TGSqlParser(EDbVendor.dbvclickhouse); 759 sub.sqltext = rest; 760 sub.tokenizeSqltext(); 761 TSourceTokenList lexed = sub.getSourcetokenlist(); 762 for (int k = 0; k < lexed.size(); k++) { 763 // the rest is one line: its offsets are columns after the ';' 764 TSourceToken c = lexed.get(k); 765 c.lineNo = t.lineNo; 766 c.columnNo = t.columnNo + semi + 1 + c.offset; 767 c.offset = t.offset + semi + 1 + c.offset; 768 c.container = sourcetokenlist; 769 c.setDbvendor(t.getDbvendor()); 770 sourcetokenlist.add(at++, c); 771 } 772 } 773 for (int k = j; k < sourcetokenlist.size(); k++) { 774 sourcetokenlist.get(k).posinlist = k; 775 } 776 } 777 778 /** 779 * Check if next token is '(' for parameterized types and emit matching tokens, 780 * then close with ')' for CAST. 781 */ 782 private void emitTypeParamsAndClose(int line, int col) { 783 TSourceToken peekToken = getanewsourcetoken(); 784 if (peekToken != null && peekToken.tokencode == 40) { // '(' 785 sourcetokenlist.add(peekToken); 786 int depth = 1; 787 while (depth > 0) { 788 TSourceToken innerToken = getanewsourcetoken(); 789 if (innerToken == null) break; 790 sourcetokenlist.add(innerToken); 791 if (innerToken.tokencode == 40) depth++; 792 else if (innerToken.tokencode == 41) depth--; 793 } 794 sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col)); 795 } else { 796 sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col)); 797 // The peeked token must be put back into the stream; 798 // since we can't push back, check if it needs further processing 799 if (peekToken != null) { 800 // Check if the peeked token is also a typecast (chained casts like x::A::B) 801 if (peekToken.tokencode == TBaseType.typecast) { 802 rewriteStandaloneTypecast(peekToken); 803 } else { 804 sourcetokenlist.add(peekToken); 805 } 806 } 807 } 808 } 809 810 /** 811 * Create a synthetic token for CAST() rewriting. 812 */ 813 private TSourceToken createSyntheticToken(String text, int tokencode, ETokenType tokentype, int lineNo, int columnNo) { 814 TSourceToken token = new TSourceToken(text); 815 token.tokencode = tokencode; 816 token.tokentype = tokentype; 817 token.tokenstatus = ETokenStatus.tsoriginal; 818 token.lineNo = lineNo; 819 token.columnNo = columnNo; 820 token.container = sourcetokenlist; 821 token.posinlist = sourcetokenlist.size(); 822 return token; 823 } 824 825 /** 826 * Handle standalone :: typecast (lexer correctly tokenized :: as typecast, 827 * e.g., after string literals or closing parentheses). 828 * The expression before :: is already in sourcetokenlist. 829 * Rewrites: ... expr :: TypeName ... → ... CAST( expr AS TypeName ) ... 830 */ 831 private void rewriteStandaloneTypecast(TSourceToken typecastToken) { 832 int line = (int) typecastToken.lineNo; 833 int col = (int) typecastToken.columnNo; 834 835 // Find the preceding expression's last solid token 836 int exprEndIdx = sourcetokenlist.size() - 1; 837 while (exprEndIdx >= 0) { 838 TSourceToken t = sourcetokenlist.get(exprEndIdx); 839 if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn 840 && t.tokentype != ETokenType.ttsimplecomment && t.tokentype != ETokenType.ttbracketedcomment) { 841 break; 842 } 843 exprEndIdx--; 844 } 845 if (exprEndIdx < 0) return; 846 847 // Find expression start 848 int exprStartIdx = exprEndIdx; 849 TSourceToken exprEndToken = sourcetokenlist.get(exprEndIdx); 850 851 if (exprEndToken.tokentype == ETokenType.ttrightparenthesis) { 852 // Walk back to find matching '(' 853 int depth = 1; 854 int idx = exprEndIdx - 1; 855 while (idx >= 0 && depth > 0) { 856 TSourceToken t = sourcetokenlist.get(idx); 857 if (t.tokentype == ETokenType.ttrightparenthesis) depth++; 858 else if (t.tokentype == ETokenType.ttleftparenthesis) depth--; 859 idx--; 860 } 861 exprStartIdx = idx + 1; // idx+1 because we decremented one extra 862 // Check for function name before '(' 863 if (exprStartIdx > 0) { 864 int fnIdx = exprStartIdx - 1; 865 while (fnIdx >= 0) { 866 TSourceToken t = sourcetokenlist.get(fnIdx); 867 if (t.tokentype != ETokenType.ttwhitespace && t.tokentype != ETokenType.ttreturn) { 868 break; 869 } 870 fnIdx--; 871 } 872 if (fnIdx >= 0) { 873 TSourceToken fnToken = sourcetokenlist.get(fnIdx); 874 if (fnToken.tokentype == ETokenType.ttidentifier || fnToken.tokentype == ETokenType.ttkeyword) { 875 exprStartIdx = fnIdx; 876 } 877 } 878 } 879 } 880 881 // Collect expression tokens and whitespace 882 List<TSourceToken> beforeExpr = new ArrayList<>(); 883 for (int i = 0; i < exprStartIdx; i++) { 884 beforeExpr.add(sourcetokenlist.get(i)); 885 } 886 // Collect whitespace just before expression that should go before CAST 887 List<TSourceToken> wsBeforeExpr = new ArrayList<>(); 888 while (!beforeExpr.isEmpty()) { 889 TSourceToken last = beforeExpr.get(beforeExpr.size() - 1); 890 if (last.tokentype == ETokenType.ttwhitespace || last.tokentype == ETokenType.ttreturn) { 891 wsBeforeExpr.add(0, beforeExpr.remove(beforeExpr.size() - 1)); 892 } else { 893 break; 894 } 895 } 896 897 List<TSourceToken> exprTokens = new ArrayList<>(); 898 for (int i = exprStartIdx; i <= exprEndIdx; i++) { 899 exprTokens.add(sourcetokenlist.get(i)); 900 } 901 List<TSourceToken> afterExpr = new ArrayList<>(); 902 for (int i = exprEndIdx + 1; i < sourcetokenlist.size(); i++) { 903 afterExpr.add(sourcetokenlist.get(i)); 904 } 905 906 // Clear and rebuild sourcetokenlist 907 sourcetokenlist.clear(); 908 sourcetokenlist.curpos = -1; 909 910 // Re-add tokens before expression (without trailing whitespace) 911 for (TSourceToken t : beforeExpr) { 912 t.posinlist = sourcetokenlist.size(); 913 sourcetokenlist.add(t); 914 } 915 916 // Add whitespace before CAST 917 for (TSourceToken t : wsBeforeExpr) { 918 t.posinlist = sourcetokenlist.size(); 919 sourcetokenlist.add(t); 920 } 921 922 // Emit CAST( 923 sourcetokenlist.add(createSyntheticToken("CAST", 750, ETokenType.ttkeyword, line, col)); 924 sourcetokenlist.add(createSyntheticToken("(", 40, ETokenType.ttleftparenthesis, line, col)); 925 926 // Re-add expression tokens 927 for (TSourceToken t : exprTokens) { 928 t.posinlist = sourcetokenlist.size(); 929 sourcetokenlist.add(t); 930 } 931 932 // Emit AS 933 sourcetokenlist.add(createSyntheticToken("AS", 341, ETokenType.ttkeyword, line, col)); 934 935 // Read type name and emit closing paren 936 TSourceToken typeToken = getanewsourcetoken(); 937 if (typeToken == null) { 938 sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col)); 939 return; 940 } 941 942 // Skip whitespace 943 while (typeToken != null && (typeToken.tokentype == ETokenType.ttwhitespace || typeToken.tokentype == ETokenType.ttreturn)) { 944 typeToken = getanewsourcetoken(); 945 } 946 if (typeToken == null) { 947 sourcetokenlist.add(createSyntheticToken(")", 41, ETokenType.ttrightparenthesis, line, col)); 948 return; 949 } 950 951 sourcetokenlist.add(typeToken); 952 953 // Check for parameterized types and close 954 emitTypeParamsAndClose(line, col); 955 } 956 957 /** 958 * Get previous non-whitespace token. 959 */ 960 private TSourceToken getprevsolidtoken(TSourceToken ptoken) { 961 TSourceToken ret = null; 962 TSourceTokenList lctokenlist = ptoken.container; 963 964 if (lctokenlist != null) { 965 if ((ptoken.posinlist > 0) && (lctokenlist.size() > ptoken.posinlist - 1)) { 966 if (!( 967 (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttwhitespace) 968 || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttreturn) 969 || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttsimplecomment) 970 || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttbracketedcomment) 971 )) { 972 ret = lctokenlist.get(ptoken.posinlist - 1); 973 } else { 974 ret = lctokenlist.nextsolidtoken(ptoken.posinlist - 1, -1, false); 975 } 976 } 977 } 978 return ret; 979 } 980 981 // ========== ClickHouse-Specific Raw Statement Extraction ========== 982 983 /** 984 * Extract raw ClickHouse SQL statements from tokenized source. 985 * <p> 986 * Simplified from MySQL - no stored procedure states, 987 * no DELIMITER command, no custom delimiters. 988 * ClickHouse uses semicolon as the only delimiter. 989 */ 990 private void doclickhousegetrawsqlstatements(SqlParseResult.Builder builder) { 991 if (TBaseType.assigned(sqlstatements)) sqlstatements.clear(); 992 if (!TBaseType.assigned(sourcetokenlist)) { 993 builder.sqlStatements(this.sqlstatements); 994 builder.errorCode(1); 995 builder.errorMessage("No source token list available"); 996 return; 997 } 998 999 TCustomSqlStatement gcurrentsqlstatement = null; 1000 EFindSqlStateType gst = EFindSqlStateType.stnormal; 1001 1002 for (int i = 0; i < sourcetokenlist.size(); i++) { 1003 TSourceToken ast = sourcetokenlist.get(i); 1004 sourcetokenlist.curpos = i; 1005 1006 // Token transformations during raw statement extraction 1007 performRawStatementTokenTransformations(ast); 1008 1009 switch (gst) { 1010 case sterror: { 1011 if (parallelWithSeparators.contains(ast)) { 1012 // PARALLEL WITH: ends the statement without joining it 1013 onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder); 1014 appendToken(gcurrentsqlstatement, ast); 1015 gst = EFindSqlStateType.stnormal; 1016 } else if (ast.tokentype == ETokenType.ttsemicolon) { 1017 appendToken(gcurrentsqlstatement, ast); 1018 onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder); 1019 gst = EFindSqlStateType.stnormal; 1020 } else { 1021 appendToken(gcurrentsqlstatement, ast); 1022 } 1023 break; 1024 } 1025 1026 case stnormal: { 1027 if ((ast.tokencode == TBaseType.cmtdoublehyphen) 1028 || (ast.tokencode == TBaseType.cmtslashstar) 1029 || (ast.tokencode == TBaseType.lexspace) 1030 || (ast.tokencode == TBaseType.lexnewline) 1031 || (ast.tokentype == ETokenType.ttsemicolon) 1032 || parallelWithSeparators.contains(ast)) { 1033 if (gcurrentsqlstatement != null) { 1034 appendToken(gcurrentsqlstatement, ast); 1035 } 1036 continue; 1037 } 1038 1039 // Find a token to start sql mode 1040 gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement); 1041 1042 if (gcurrentsqlstatement != null) { 1043 gst = EFindSqlStateType.stsql; 1044 appendToken(gcurrentsqlstatement, ast); 1045 } else { 1046 // Error token found 1047 this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo, 1048 (ast.columnNo < 0 ? 0 : ast.columnNo), 1049 "Error when tokenize", EErrorType.spwarning, 1050 TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist)); 1051 1052 ast.tokentype = ETokenType.tttokenlizererrortoken; 1053 gst = EFindSqlStateType.sterror; 1054 1055 gcurrentsqlstatement = new TUnknownSqlStatement(vendor); 1056 gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid; 1057 appendToken(gcurrentsqlstatement, ast); 1058 } 1059 1060 break; 1061 } 1062 1063 case stsql: { 1064 if (parallelWithSeparators.contains(ast)) { 1065 // PARALLEL WITH (clickhousePostTokenAdjust): ends the 1066 // statement without joining it, like the text between 1067 // two statements. 1068 gst = EFindSqlStateType.stnormal; 1069 onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder); 1070 appendToken(gcurrentsqlstatement, ast); 1071 continue; 1072 } 1073 if (ast.tokentype == ETokenType.ttsemicolon) { 1074 gst = EFindSqlStateType.stnormal; 1075 appendToken(gcurrentsqlstatement, ast); 1076 gcurrentsqlstatement.semicolonended = ast; 1077 onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder); 1078 continue; 1079 } 1080 1081 if (ast.tokencode == TBaseType.cmtdoublehyphen) { 1082 if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)) { 1083 gst = EFindSqlStateType.stnormal; 1084 onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder); 1085 continue; 1086 } 1087 } 1088 1089 appendToken(gcurrentsqlstatement, ast); 1090 break; 1091 } 1092 1093 default: 1094 break; 1095 } 1096 } 1097 1098 // Last statement 1099 if ((gcurrentsqlstatement != null) && 1100 ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.sterror))) { 1101 onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, true, builder); 1102 } 1103 1104 // Populate builder with results 1105 builder.sqlStatements(this.sqlstatements); 1106 builder.syntaxErrors(syntaxErrors instanceof ArrayList ? 1107 (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors)); 1108 builder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size()); 1109 } 1110 1111 /** 1112 * Handle token transformations during raw statement extraction. 1113 * <p> 1114 * Handles MySQL-inherited token transformations for DATE, TIME, TIMESTAMP keywords 1115 * and other context-dependent token adjustments. 1116 */ 1117 private void performRawStatementTokenTransformations(TSourceToken ast) { 1118 if (ast.tokencode == TBaseType.rrw_date) { 1119 TSourceToken st1 = ast.nextSolidToken(); 1120 if (st1 != null) { 1121 if (st1.tokencode == '(') { 1122 ast.tokencode = TBaseType.rrw_mysql_date_function; 1123 } else if (st1.tokencode == TBaseType.sconst) { 1124 ast.tokencode = TBaseType.rrw_mysql_date_const; 1125 } 1126 } 1127 } else if (ast.tokencode == TBaseType.rrw_time) { 1128 TSourceToken st1 = ast.nextSolidToken(); 1129 if (st1 != null) { 1130 if (st1.tokencode == TBaseType.sconst) { 1131 ast.tokencode = TBaseType.rrw_mysql_time_const; 1132 } 1133 } 1134 } else if (ast.tokencode == TBaseType.rrw_timestamp) { 1135 TSourceToken st1 = ast.nextSolidToken(); 1136 if (st1 != null) { 1137 if (st1.tokencode == TBaseType.sconst) { 1138 ast.tokencode = TBaseType.rrw_mysql_timestamp_constant; 1139 } else if (st1.tokencode == TBaseType.ident) { 1140 if (st1.toString().startsWith("\"")) { 1141 ast.tokencode = TBaseType.rrw_mysql_timestamp_constant; 1142 st1.tokencode = TBaseType.sconst; 1143 } 1144 } 1145 } 1146 } else if (ast.tokencode == TBaseType.rrw_mysql_position) { 1147 TSourceToken st1 = ast.nextSolidToken(); 1148 if (st1 != null) { 1149 if (st1.tokencode != '(') { 1150 ast.tokencode = TBaseType.ident; 1151 } 1152 } 1153 } else if (ast.tokencode == TBaseType.rrw_interval) { 1154 TSourceToken leftParen = ast.searchToken('(', 1); 1155 if (leftParen != null) { 1156 int k = leftParen.posinlist + 1; 1157 int nested = 1; 1158 boolean commaToken = false; 1159 while (k < ast.container.size()) { 1160 if (ast.container.get(k).tokencode == '(') { 1161 nested++; 1162 } 1163 if (ast.container.get(k).tokencode == ')') { 1164 nested--; 1165 if (nested == 0) break; 1166 } 1167 if ((ast.container.get(k).tokencode == ',') && (nested == 1)) { 1168 commaToken = true; 1169 break; 1170 } 1171 k++; 1172 } 1173 if (commaToken) { 1174 ast.tokencode = TBaseType.rrw_mysql_interval_func; 1175 } 1176 } 1177 } 1178 } 1179 1180 private void appendToken(TCustomSqlStatement statement, TSourceToken token) { 1181 if (statement == null || token == null) { 1182 return; 1183 } 1184 token.stmt = statement; 1185 statement.sourcetokenlist.add(token); 1186 } 1187 1188 @Override 1189 public String toString() { 1190 return "ClickhouseSqlParser{vendor=" + vendor + "}"; 1191 } 1192}