Source code

001package gudusoft.gsqlparser.parser;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.TBaseType;
005import gudusoft.gsqlparser.TCustomLexer;
006import gudusoft.gsqlparser.TCustomParser;
007import gudusoft.gsqlparser.TCustomSqlStatement;
008import gudusoft.gsqlparser.TLexerStarrocks;
009import gudusoft.gsqlparser.TParserStarrocksSql;
010import gudusoft.gsqlparser.TSourceToken;
011import gudusoft.gsqlparser.TSourceTokenList;
012import gudusoft.gsqlparser.TStatementList;
013import gudusoft.gsqlparser.TSyntaxError;
014import gudusoft.gsqlparser.EFindSqlStateType;
015import gudusoft.gsqlparser.ETokenType;
016import gudusoft.gsqlparser.ETokenStatus;
017import gudusoft.gsqlparser.ESqlStatementType;
018import gudusoft.gsqlparser.EErrorType;
019import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
020import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
021import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
022import gudusoft.gsqlparser.compiler.TContext;
023import gudusoft.gsqlparser.sqlenv.TSQLEnv;
024import gudusoft.gsqlparser.compiler.TGlobalScope;
025import gudusoft.gsqlparser.compiler.TFrame;
026
027import java.io.BufferedReader;
028import java.util.ArrayList;
029import java.util.List;
030import java.util.Stack;
031
032/**
033 * StarRocks SQL parser implementation.
034 *
035 * <p>StarRocks is a fork of Apache Doris (from 2020) and maintains MySQL protocol
036 * compatibility. This parser is based on the Doris parser with extensions for
037 * StarRocks-specific features including:
038 * <ul>
039 *   <li>OLAP data models (DUPLICATE KEY, AGGREGATE KEY, UNIQUE KEY, PRIMARY KEY)</li>
040 *   <li>DISTRIBUTED BY HASH/RANDOM clause</li>
041 *   <li>PROPERTIES clause</li>
042 *   <li>StarRocks-specific data types (LARGEINT, HLL, BITMAP, JSON, ARRAY, MAP, STRUCT)</li>
043 *   <li>External catalogs (CREATE EXTERNAL CATALOG)</li>
044 *   <li>Asynchronous materialized views (REFRESH ASYNC/MANUAL)</li>
045 *   <li>SUBMIT TASK for async ETL</li>
046 * </ul>
047 *
048 * <p><b>Design Notes:</b>
049 * <ul>
050 *   <li>Extends {@link AbstractSqlParser}</li>
051 *   <li>Uses {@link TLexerStarrocks} and {@link TParserStarrocksSql}</li>
052 *   <li>Delimiter character: ';'</li>
053 * </ul>
054 *
055 * @see SqlParser
056 * @see AbstractSqlParser
057 * @see TLexerStarrocks
058 * @see TParserStarrocksSql
059 * @since 4.0.2.0
060 */
061public class StarrocksSqlParser extends AbstractSqlParser {
062
063    // ========== Lexer and Parser Instances ==========
064
065    /** The StarRocks lexer used for tokenization (public for TGSqlParser.getFlexer()) */
066    public TLexerStarrocks flexer;
067    private TParserStarrocksSql fparser;
068
069    // ========== State Variables for Raw Statement Extraction ==========
070    private String userDelimiterStr;
071    private char curdelimiterchar;
072    private boolean waitingDelimiter;
073
074    // ========== Constructor ==========
075
076    /**
077     * Construct StarRocks SQL parser.
078     * <p>
079     * Configures the parser for StarRocks database with default delimiter: semicolon (;)
080     */
081    public StarrocksSqlParser() {
082        super(EDbVendor.dbvstarrocks);
083
084        // Set delimiter character - StarRocks uses semicolon like standard MySQL
085        this.delimiterChar = ';';
086        this.defaultDelimiterStr = ";";
087
088        // Create lexer once - will be reused for all parsing operations
089        this.flexer = new TLexerStarrocks();
090        this.flexer.delimiterchar = this.delimiterChar;
091        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;
092
093        // CRITICAL: Set lexer for inherited getanewsourcetoken() method
094        this.lexer = this.flexer;
095
096        // Create parser once - will be reused for all parsing operations
097        this.fparser = new TParserStarrocksSql(null);
098        this.fparser.lexer = this.flexer;
099    }
100
101    // ========== AbstractSqlParser Abstract Methods Implementation ==========
102
103    @Override
104    protected TCustomLexer getLexer(ParserContext context) {
105        return this.flexer;
106    }
107
108    @Override
109    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
110        this.fparser.sourcetokenlist = tokens;
111        return this.fparser;
112    }
113
114    @Override
115    protected void tokenizeVendorSql() {
116        dostarrockstexttotokenlist();
117    }
118
119    @Override
120    protected void setupVendorParsersForExtraction() {
121        this.fparser.sqlcmds = this.sqlcmds;
122        this.fparser.sourcetokenlist = this.sourcetokenlist;
123    }
124
125    @Override
126    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
127        dostarrocksgetrawsqlstatements(builder);
128    }
129
130    @Override
131    protected TStatementList performParsing(ParserContext context,
132                                           TCustomParser parser,
133                                           TCustomParser secondaryParser,
134                                           TSourceTokenList tokens,
135                                           TStatementList rawStatements) {
136        this.sourcetokenlist = tokens;
137        this.parserContext = context;
138        this.sqlstatements = rawStatements;
139
140        this.sqlcmds = SqlCmdsFactory.get(vendor);
141        this.fparser.sqlcmds = this.sqlcmds;
142
143        initializeGlobalContext();
144
145        for (int i = 0; i < sqlstatements.size(); i++) {
146            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);
147
148            try {
149                stmt.setFrameStack(frameStack);
150                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());
151
152                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
153                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
154                    handleCreateTableErrorRecovery(stmt);
155                }
156
157                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
158                    copyErrorsFromStatement(stmt);
159                }
160            } catch (Exception ex) {
161                handleStatementParsingException(stmt, i, ex);
162                continue;
163            }
164        }
165
166        if (globalFrame != null) {
167            globalFrame.popMeFromStack(frameStack);
168        }
169
170        return this.sqlstatements;
171    }
172
173    private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
174        if ((stmt.sqlstatementtype != ESqlStatementType.sstcreatetable) || TBaseType.c_createTableStrictParsing) {
175            return;
176        }
177
178        // A CREATE TABLE AS SELECT keeps its query; the recovery below would drop it or cut it short.
179        int queryStart = CreateTableAsSelectRecovery.queryStart(stmt.sourcetokenlist);
180        if (queryStart >= 0) {
181            if (CreateTableAsSelectRecovery.ignoreClausesBeforeQuery(stmt.sourcetokenlist, queryStart)) {
182                stmt.clearError();
183                stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree());
184            }
185            return;
186        }
187
188        int nested = 0;
189        boolean isIgnore = false, isFoundIgnoreToken = false;
190        TSourceToken firstIgnoreToken = null;
191
192        for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
193            TSourceToken st = stmt.sourcetokenlist.get(k);
194            if (isIgnore) {
195                if (st.issolidtoken() && (st.tokencode != ';')) {
196                    isFoundIgnoreToken = true;
197                    if (firstIgnoreToken == null) {
198                        firstIgnoreToken = st;
199                    }
200                }
201                if (st.tokencode != ';') {
202                    st.tokencode = TBaseType.sqlpluscmd;
203                }
204                continue;
205            }
206            if (st.tokencode == (int) ')') {
207                nested--;
208                if (nested == 0) {
209                    boolean isSelect = false;
210                    TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
211                    if (st1 != null) {
212                        TSourceToken st2 = st.searchToken((int) '(', 2);
213                        if (st2 != null) {
214                            TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
215                            isSelect = (st3 != null);
216                        }
217                    }
218                    if (!isSelect) isIgnore = true;
219                }
220            } else if (st.tokencode == (int) '(') {
221                nested++;
222            }
223        }
224
225        if (isFoundIgnoreToken) {
226            stmt.clearError();
227            stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree());
228        }
229    }
230
231    // ========== StarRocks-Specific Tokenization ==========
232
233    private void dostarrockstexttotokenlist() {
234        TSourceToken asourcetoken, lcprevst;
235        int yychar;
236
237        asourcetoken = getanewsourcetoken();
238        if (asourcetoken == null) return;
239        yychar = asourcetoken.tokencode;
240
241        while (yychar > 0) {
242            sourcetokenlist.add(asourcetoken);
243            asourcetoken = getanewsourcetoken();
244            if (asourcetoken == null) break;
245
246            if (asourcetoken.tokencode == TBaseType.rrw_rollup) {
247                lcprevst = getprevsolidtoken(asourcetoken);
248                if (lcprevst != null) {
249                    if (lcprevst.tokencode == TBaseType.rrw_with)
250                        lcprevst.tokencode = TBaseType.with_rollup;
251                }
252            }
253
254            yychar = asourcetoken.tokencode;
255        }
256    }
257
258    private TSourceToken getprevsolidtoken(TSourceToken ptoken) {
259        TSourceToken ret = null;
260        TSourceTokenList lctokenlist = ptoken.container;
261
262        if (lctokenlist != null) {
263            if ((ptoken.posinlist > 0) && (lctokenlist.size() > ptoken.posinlist - 1)) {
264                if (!(
265                        (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttwhitespace)
266                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttreturn)
267                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttsimplecomment)
268                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttbracketedcomment)
269                )) {
270                    ret = lctokenlist.get(ptoken.posinlist - 1);
271                } else {
272                    ret = lctokenlist.nextsolidtoken(ptoken.posinlist - 1, -1, false);
273                }
274            }
275        }
276        return ret;
277    }
278
279    // ========== StarRocks-Specific Raw Statement Extraction ==========
280
281    private void dostarrocksgetrawsqlstatements(SqlParseResult.Builder builder) {
282        TCustomSqlStatement gcurrentsqlstatement = null;
283        EFindSqlStateType gst = EFindSqlStateType.stnormal;
284
285        userDelimiterStr = defaultDelimiterStr;
286
287        if (TBaseType.assigned(sqlstatements)) sqlstatements.clear();
288        if (!TBaseType.assigned(sourcetokenlist)) {
289            builder.sqlStatements(this.sqlstatements);
290            builder.errorCode(1);
291            builder.errorMessage("No source token list available");
292            return;
293        }
294
295        for (int i = 0; i < sourcetokenlist.size(); i++) {
296            TSourceToken ast = sourcetokenlist.get(i);
297            sourcetokenlist.curpos = i;
298
299            performRawStatementTokenTransformations(ast);
300
301            switch (gst) {
302                case sterror: {
303                    if (ast.tokentype == ETokenType.ttsemicolon) {
304                        appendToken(gcurrentsqlstatement, ast);
305                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
306                        gst = EFindSqlStateType.stnormal;
307                    } else {
308                        appendToken(gcurrentsqlstatement, ast);
309                    }
310                    break;
311                }
312
313                case stnormal: {
314                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
315                            || (ast.tokencode == TBaseType.cmtslashstar)
316                            || (ast.tokencode == TBaseType.lexspace)
317                            || (ast.tokencode == TBaseType.lexnewline)
318                            || (ast.tokentype == ETokenType.ttsemicolon)) {
319                        if (TBaseType.assigned(gcurrentsqlstatement)) {
320                            appendToken(gcurrentsqlstatement, ast);
321                        }
322                        continue;
323                    }
324
325                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);
326
327                    if (TBaseType.assigned(gcurrentsqlstatement)) {
328                        gst = EFindSqlStateType.stsql;
329                        appendToken(gcurrentsqlstatement, ast);
330                    }
331
332                    if (!TBaseType.assigned(gcurrentsqlstatement)) {
333                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo, (ast.columnNo < 0 ? 0 : ast.columnNo),
334                                "Error when tokenize", EErrorType.spwarning, TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));
335
336                        ast.tokentype = ETokenType.tttokenlizererrortoken;
337                        gst = EFindSqlStateType.sterror;
338
339                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
340                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
341                        appendToken(gcurrentsqlstatement, ast);
342                    }
343                    break;
344                }
345
346                case stsql: {
347                    if (ast.tokentype == ETokenType.ttsemicolon) {
348                        gst = EFindSqlStateType.stnormal;
349                        appendToken(gcurrentsqlstatement, ast);
350                        gcurrentsqlstatement.semicolonended = ast;
351                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
352                        continue;
353                    }
354
355                    if (ast.tokencode == TBaseType.cmtdoublehyphen) {
356                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)) {
357                            gst = EFindSqlStateType.stnormal;
358                            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
359                            continue;
360                        }
361                    }
362
363                    appendToken(gcurrentsqlstatement, ast);
364                    break;
365                }
366
367                default:
368                    break;
369            }
370        }
371
372        // Last statement
373        if (TBaseType.assigned(gcurrentsqlstatement) && ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.sterror))) {
374            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, true, builder);
375        }
376
377        builder.sqlStatements(this.sqlstatements);
378        builder.syntaxErrors(syntaxErrors instanceof ArrayList ?
379                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
380        builder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size());
381    }
382
383    private void performRawStatementTokenTransformations(TSourceToken ast) {
384        // StarRocks-specific token transformations (MySQL-compatible)
385        if (ast.tokencode == TBaseType.rrw_date) {
386            TSourceToken st1 = ast.nextSolidToken();
387            if (st1 != null) {
388                if (st1.tokencode == '(') {
389                    ast.tokencode = TBaseType.rrw_mysql_date_function;
390                } else if (st1.tokencode == TBaseType.sconst) {
391                    ast.tokencode = TBaseType.rrw_mysql_date_const;
392                }
393            }
394        } else if (ast.tokencode == TBaseType.rrw_time) {
395            TSourceToken st1 = ast.nextSolidToken();
396            if (st1 != null) {
397                if (st1.tokencode == TBaseType.sconst) {
398                    ast.tokencode = TBaseType.rrw_mysql_time_const;
399                }
400            }
401        } else if (ast.tokencode == TBaseType.rrw_timestamp) {
402            TSourceToken st1 = ast.nextSolidToken();
403            if (st1 != null) {
404                if (st1.tokencode == TBaseType.sconst) {
405                    ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
406                }
407            }
408        }
409    }
410
411    private void appendToken(TCustomSqlStatement statement, TSourceToken token) {
412        if (statement == null || token == null) {
413            return;
414        }
415        token.stmt = statement;
416        statement.sourcetokenlist.add(token);
417    }
418
419    @Override
420    public String toString() {
421        return "StarrocksSqlParser{vendor=" + vendor + "}";
422    }
423}