Source code

001package gudusoft.gsqlparser.parser;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.TBaseType;
005import gudusoft.gsqlparser.TCustomLexer;
006import gudusoft.gsqlparser.TCustomParser;
007import gudusoft.gsqlparser.TCustomSqlStatement;
008import gudusoft.gsqlparser.TLexerDoris;
009import gudusoft.gsqlparser.TParserDoris;
010import gudusoft.gsqlparser.TSourceToken;
011import gudusoft.gsqlparser.TSourceTokenList;
012import gudusoft.gsqlparser.TStatementList;
013import gudusoft.gsqlparser.TSyntaxError;
014import gudusoft.gsqlparser.EFindSqlStateType;
015import gudusoft.gsqlparser.ETokenType;
016import gudusoft.gsqlparser.ETokenStatus;
017import gudusoft.gsqlparser.ESqlStatementType;
018import gudusoft.gsqlparser.EErrorType;
019import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
020import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
021import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
022import gudusoft.gsqlparser.compiler.TContext;
023import gudusoft.gsqlparser.sqlenv.TSQLEnv;
024import gudusoft.gsqlparser.compiler.TGlobalScope;
025import gudusoft.gsqlparser.compiler.TFrame;
026
027import java.io.BufferedReader;
028import java.util.ArrayList;
029import java.util.List;
030import java.util.Stack;
031
032/**
033 * Apache Doris SQL parser implementation.
034 *
035 * <p>Doris is MySQL-compatible, so this parser is based on the MySQL parser
036 * with extensions for Doris-specific features including:
037 * <ul>
038 *   <li>OLAP data models (DUPLICATE KEY, AGGREGATE KEY, UNIQUE KEY)</li>
039 *   <li>DISTRIBUTED BY clause</li>
040 *   <li>PROPERTIES clause</li>
041 *   <li>Doris-specific data types (LARGEINT, HLL, BITMAP, etc.)</li>
042 *   <li>Load and export statements (BROKER LOAD, EXPORT, ROUTINE LOAD)</li>
043 * </ul>
044 *
045 * <p><b>Design Notes:</b>
046 * <ul>
047 *   <li>Extends {@link AbstractSqlParser}</li>
048 *   <li>Uses {@link TLexerDoris} and {@link TParserDoris}</li>
049 *   <li>Delimiter character: ';'</li>
050 * </ul>
051 *
052 * @see SqlParser
053 * @see AbstractSqlParser
054 * @see TLexerDoris
055 * @see TParserDoris
056 * @since 3.2.0.0
057 */
058public class DorisSqlParser extends AbstractSqlParser {
059
060    // ========== Lexer and Parser Instances ==========
061
062    /** The Doris lexer used for tokenization (public for TGSqlParser.getFlexer()) */
063    public TLexerDoris flexer;
064    private TParserDoris fparser;
065
066    // ========== State Variables for Raw Statement Extraction ==========
067    private String userDelimiterStr;
068    private char curdelimiterchar;
069    private boolean waitingDelimiter;
070
071    // ========== Constructor ==========
072
073    /**
074     * Construct Doris SQL parser.
075     * <p>
076     * Configures the parser for Apache Doris database with default delimiter: semicolon (;)
077     */
078    public DorisSqlParser() {
079        super(EDbVendor.dbvdoris);
080
081        // Set delimiter character - Doris uses semicolon like standard MySQL
082        this.delimiterChar = ';';
083        this.defaultDelimiterStr = ";";
084
085        // Create lexer once - will be reused for all parsing operations
086        this.flexer = new TLexerDoris();
087        this.flexer.delimiterchar = this.delimiterChar;
088        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;
089
090        // CRITICAL: Set lexer for inherited getanewsourcetoken() method
091        this.lexer = this.flexer;
092
093        // Create parser once - will be reused for all parsing operations
094        this.fparser = new TParserDoris(null);
095        this.fparser.lexer = this.flexer;
096    }
097
098    // ========== AbstractSqlParser Abstract Methods Implementation ==========
099
100    @Override
101    protected TCustomLexer getLexer(ParserContext context) {
102        return this.flexer;
103    }
104
105    @Override
106    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
107        this.fparser.sourcetokenlist = tokens;
108        return this.fparser;
109    }
110
111    @Override
112    protected void tokenizeVendorSql() {
113        dodoristexttotokenlist();
114    }
115
116    @Override
117    protected void setupVendorParsersForExtraction() {
118        this.fparser.sqlcmds = this.sqlcmds;
119        this.fparser.sourcetokenlist = this.sourcetokenlist;
120    }
121
122    @Override
123    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
124        dodorisgetrawsqlstatements(builder);
125    }
126
127    @Override
128    protected TStatementList performParsing(ParserContext context,
129                                           TCustomParser parser,
130                                           TCustomParser secondaryParser,
131                                           TSourceTokenList tokens,
132                                           TStatementList rawStatements) {
133        this.sourcetokenlist = tokens;
134        this.parserContext = context;
135        this.sqlstatements = rawStatements;
136
137        this.sqlcmds = SqlCmdsFactory.get(vendor);
138        this.fparser.sqlcmds = this.sqlcmds;
139
140        initializeGlobalContext();
141
142        for (int i = 0; i < sqlstatements.size(); i++) {
143            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);
144
145            try {
146                stmt.setFrameStack(frameStack);
147                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());
148
149                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
150                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
151                    handleCreateTableErrorRecovery(stmt);
152                }
153
154                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
155                    copyErrorsFromStatement(stmt);
156                }
157            } catch (Exception ex) {
158                handleStatementParsingException(stmt, i, ex);
159                continue;
160            }
161        }
162
163        if (globalFrame != null) {
164            globalFrame.popMeFromStack(frameStack);
165        }
166
167        return this.sqlstatements;
168    }
169
170    private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
171        if ((stmt.sqlstatementtype != ESqlStatementType.sstcreatetable) || TBaseType.c_createTableStrictParsing) {
172            return;
173        }
174
175        // A CREATE TABLE AS SELECT keeps its query; the recovery below would drop it or cut it short.
176        int queryStart = CreateTableAsSelectRecovery.queryStart(stmt.sourcetokenlist);
177        if (queryStart >= 0) {
178            if (CreateTableAsSelectRecovery.ignoreClausesBeforeQuery(stmt.sourcetokenlist, queryStart)) {
179                stmt.clearError();
180                stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree());
181            }
182            return;
183        }
184
185        int nested = 0;
186        boolean isIgnore = false, isFoundIgnoreToken = false;
187        TSourceToken firstIgnoreToken = null;
188
189        for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
190            TSourceToken st = stmt.sourcetokenlist.get(k);
191            if (isIgnore) {
192                if (st.issolidtoken() && (st.tokencode != ';')) {
193                    isFoundIgnoreToken = true;
194                    if (firstIgnoreToken == null) {
195                        firstIgnoreToken = st;
196                    }
197                }
198                if (st.tokencode != ';') {
199                    st.tokencode = TBaseType.sqlpluscmd;
200                }
201                continue;
202            }
203            if (st.tokencode == (int) ')') {
204                nested--;
205                if (nested == 0) {
206                    boolean isSelect = false;
207                    TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
208                    if (st1 != null) {
209                        TSourceToken st2 = st.searchToken((int) '(', 2);
210                        if (st2 != null) {
211                            TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
212                            isSelect = (st3 != null);
213                        }
214                    }
215                    if (!isSelect) isIgnore = true;
216                }
217            } else if (st.tokencode == (int) '(') {
218                nested++;
219            }
220        }
221
222        if (isFoundIgnoreToken) {
223            stmt.clearError();
224            stmt.parsestatement(null, false, this.parserContext.isOnlyNeedRawParseTree());
225        }
226    }
227
228    // ========== Doris-Specific Tokenization ==========
229
230    private void dodoristexttotokenlist() {
231        TSourceToken asourcetoken, lcprevst;
232        int yychar;
233
234        asourcetoken = getanewsourcetoken();
235        if (asourcetoken == null) return;
236        yychar = asourcetoken.tokencode;
237
238        while (yychar > 0) {
239            sourcetokenlist.add(asourcetoken);
240            asourcetoken = getanewsourcetoken();
241            if (asourcetoken == null) break;
242
243            if (asourcetoken.tokencode == TBaseType.rrw_rollup) {
244                lcprevst = getprevsolidtoken(asourcetoken);
245                if (lcprevst != null) {
246                    if (lcprevst.tokencode == TBaseType.rrw_with)
247                        lcprevst.tokencode = TBaseType.with_rollup;
248                }
249            }
250
251            yychar = asourcetoken.tokencode;
252        }
253    }
254
255    private TSourceToken getprevsolidtoken(TSourceToken ptoken) {
256        TSourceToken ret = null;
257        TSourceTokenList lctokenlist = ptoken.container;
258
259        if (lctokenlist != null) {
260            if ((ptoken.posinlist > 0) && (lctokenlist.size() > ptoken.posinlist - 1)) {
261                if (!(
262                        (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttwhitespace)
263                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttreturn)
264                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttsimplecomment)
265                        || (lctokenlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttbracketedcomment)
266                )) {
267                    ret = lctokenlist.get(ptoken.posinlist - 1);
268                } else {
269                    ret = lctokenlist.nextsolidtoken(ptoken.posinlist - 1, -1, false);
270                }
271            }
272        }
273        return ret;
274    }
275
276    // ========== Doris-Specific Raw Statement Extraction ==========
277
278    private void dodorisgetrawsqlstatements(SqlParseResult.Builder builder) {
279        TCustomSqlStatement gcurrentsqlstatement = null;
280        EFindSqlStateType gst = EFindSqlStateType.stnormal;
281
282        userDelimiterStr = defaultDelimiterStr;
283
284        if (TBaseType.assigned(sqlstatements)) sqlstatements.clear();
285        if (!TBaseType.assigned(sourcetokenlist)) {
286            builder.sqlStatements(this.sqlstatements);
287            builder.errorCode(1);
288            builder.errorMessage("No source token list available");
289            return;
290        }
291
292        for (int i = 0; i < sourcetokenlist.size(); i++) {
293            TSourceToken ast = sourcetokenlist.get(i);
294            sourcetokenlist.curpos = i;
295
296            performRawStatementTokenTransformations(ast);
297
298            switch (gst) {
299                case sterror: {
300                    if (ast.tokentype == ETokenType.ttsemicolon) {
301                        appendToken(gcurrentsqlstatement, ast);
302                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
303                        gst = EFindSqlStateType.stnormal;
304                    } else {
305                        appendToken(gcurrentsqlstatement, ast);
306                    }
307                    break;
308                }
309
310                case stnormal: {
311                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
312                            || (ast.tokencode == TBaseType.cmtslashstar)
313                            || (ast.tokencode == TBaseType.lexspace)
314                            || (ast.tokencode == TBaseType.lexnewline)
315                            || (ast.tokentype == ETokenType.ttsemicolon)) {
316                        if (TBaseType.assigned(gcurrentsqlstatement)) {
317                            appendToken(gcurrentsqlstatement, ast);
318                        }
319                        continue;
320                    }
321
322                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);
323
324                    if (TBaseType.assigned(gcurrentsqlstatement)) {
325                        gst = EFindSqlStateType.stsql;
326                        appendToken(gcurrentsqlstatement, ast);
327                    }
328
329                    if (!TBaseType.assigned(gcurrentsqlstatement)) {
330                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo, (ast.columnNo < 0 ? 0 : ast.columnNo),
331                                "Error when tokenize", EErrorType.spwarning, TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));
332
333                        ast.tokentype = ETokenType.tttokenlizererrortoken;
334                        gst = EFindSqlStateType.sterror;
335
336                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
337                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
338                        appendToken(gcurrentsqlstatement, ast);
339                    }
340                    break;
341                }
342
343                case stsql: {
344                    if (ast.tokentype == ETokenType.ttsemicolon) {
345                        gst = EFindSqlStateType.stnormal;
346                        appendToken(gcurrentsqlstatement, ast);
347                        gcurrentsqlstatement.semicolonended = ast;
348                        onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
349                        continue;
350                    }
351
352                    if (ast.tokencode == TBaseType.cmtdoublehyphen) {
353                        if (ast.toString().trim().endsWith(TBaseType.sqlflow_stmt_delimiter_str)) {
354                            gst = EFindSqlStateType.stnormal;
355                            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, false, builder);
356                            continue;
357                        }
358                    }
359
360                    appendToken(gcurrentsqlstatement, ast);
361                    break;
362                }
363
364                default:
365                    break;
366            }
367        }
368
369        // Last statement
370        if (TBaseType.assigned(gcurrentsqlstatement) && ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.sterror))) {
371            onRawStatementComplete(this.parserContext, gcurrentsqlstatement, this.fparser, null, this.sqlstatements, true, builder);
372        }
373
374        builder.sqlStatements(this.sqlstatements);
375        builder.syntaxErrors(syntaxErrors instanceof ArrayList ?
376                (ArrayList<TSyntaxError>) syntaxErrors : new ArrayList<>(syntaxErrors));
377        builder.errorCode(syntaxErrors.isEmpty() ? 0 : syntaxErrors.size());
378    }
379
380    private void performRawStatementTokenTransformations(TSourceToken ast) {
381        // Doris-specific token transformations can be added here
382        // For now, handle common MySQL-compatible transformations
383        if (ast.tokencode == TBaseType.rrw_date) {
384            TSourceToken st1 = ast.nextSolidToken();
385            if (st1 != null) {
386                if (st1.tokencode == '(') {
387                    ast.tokencode = TBaseType.rrw_mysql_date_function;
388                } else if (st1.tokencode == TBaseType.sconst) {
389                    ast.tokencode = TBaseType.rrw_mysql_date_const;
390                }
391            }
392        } else if (ast.tokencode == TBaseType.rrw_time) {
393            TSourceToken st1 = ast.nextSolidToken();
394            if (st1 != null) {
395                if (st1.tokencode == TBaseType.sconst) {
396                    ast.tokencode = TBaseType.rrw_mysql_time_const;
397                }
398            }
399        } else if (ast.tokencode == TBaseType.rrw_timestamp) {
400            TSourceToken st1 = ast.nextSolidToken();
401            if (st1 != null) {
402                if (st1.tokencode == TBaseType.sconst) {
403                    ast.tokencode = TBaseType.rrw_mysql_timestamp_constant;
404                }
405            }
406        } else if (ast.tokencode == TBaseType.rrw_with) {
407            // INSERT INTO t [PARTITION (...)] WITH LABEL name ...: the label names the load job, not data the
408            // statement reads or writes, and the grammar has no rule for it, so the parser skips it (Mantis 4855).
409            // A CTE named label reads WITH label AS or WITH label (, never WITH label name.
410            TSourceToken label = ast.nextSolidToken();
411            TSourceToken name = (label == null) ? null : label.nextSolidToken();
412            if ((label != null) && (name != null)
413                    && label.toString().equalsIgnoreCase("label") // non-identifier-compare: keyword LABEL
414                    && ((name.tokentype == ETokenType.ttidentifier)
415                        || ((name.tokentype == ETokenType.ttkeyword) && (name.tokencode != TBaseType.rrw_as)))
416                    && startsInsert(ast)) {
417                ast.tokenstatus = ETokenStatus.tsignorebyyacc;
418                label.tokenstatus = ETokenStatus.tsignorebyyacc;
419                name.tokenstatus = ETokenStatus.tsignorebyyacc;
420            }
421        }
422    }
423
424    /** Whether the statement {@code token} is in starts with INSERT (BEGIN WITH LABEL name is a transaction's). */
425    private static boolean startsInsert(TSourceToken token) {
426        TSourceToken first = token;
427        for (TSourceToken prev = token.container.nextsolidtoken(token, -1, false);
428             (prev != null) && (prev.tokentype != ETokenType.ttsemicolon);
429             prev = prev.container.nextsolidtoken(prev, -1, false)) {
430            first = prev;
431        }
432        return first.tokencode == TBaseType.rrw_insert;
433    }
434
435    private void appendToken(TCustomSqlStatement statement, TSourceToken token) {
436        if (statement == null || token == null) {
437            return;
438        }
439        token.stmt = statement;
440        statement.sourcetokenlist.add(token);
441    }
442
443    @Override
444    public String toString() {
445        return "DorisSqlParser{vendor=" + vendor + "}";
446    }
447}