Source code

001package gudusoft.gsqlparser.parser;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.TBaseType;
005import gudusoft.gsqlparser.TGSqlParser;
006import gudusoft.gsqlparser.TCustomLexer;
007import gudusoft.gsqlparser.TCustomParser;
008import gudusoft.gsqlparser.TCustomSqlStatement;
009import gudusoft.gsqlparser.TLexerSparksql;
010import gudusoft.gsqlparser.TParserSparksql;
011import gudusoft.gsqlparser.TSourceToken;
012import gudusoft.gsqlparser.TSourceTokenList;
013import gudusoft.gsqlparser.TStatementList;
014import gudusoft.gsqlparser.TSyntaxError;
015import gudusoft.gsqlparser.EFindSqlStateType;
016import gudusoft.gsqlparser.ETokenType;
017import gudusoft.gsqlparser.ETokenStatus;
018import gudusoft.gsqlparser.ESqlStatementType;
019import gudusoft.gsqlparser.EErrorType;
020import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
021import gudusoft.gsqlparser.stmt.mysql.TMySQLSource;
022import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
023import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
024import gudusoft.gsqlparser.compiler.TContext;
025import gudusoft.gsqlparser.sqlenv.TSQLEnv;
026import gudusoft.gsqlparser.compiler.TGlobalScope;
027import gudusoft.gsqlparser.compiler.TFrame;
028import gudusoft.gsqlparser.resolver.TSQLResolver;
029import gudusoft.gsqlparser.resolver2.TSQLResolver2;
030import gudusoft.gsqlparser.resolver2.TSQLResolverConfig;
031import gudusoft.gsqlparser.TLog;
032import gudusoft.gsqlparser.compiler.TASTEvaluator;
033
034import java.io.BufferedReader;
035import java.util.ArrayList;
036import java.util.List;
037import java.util.Stack;
038
039import static gudusoft.gsqlparser.ESqlStatementType.*;
040
041/**
042 * Apache Spark SQL parser implementation.
043 *
044 * <p>This parser handles SparkSQL-specific SQL syntax including:
045 * <ul>
046 *   <li>SparkSQL SELECT statements</li>
047 *   <li>SparkSQL-specific functions (DATE, TIME, TIMESTAMP, INTERVAL)</li>
048 *   <li>SparkSQL stored procedures and triggers</li>
049 *   <li>SparkSQL delimiter handling</li>
050 * </ul>
051 *
052 * <p><b>Design Notes:</b>
053 * <ul>
054 *   <li>Extends {@link AbstractSqlParser} using the template method pattern</li>
055 *   <li>Uses {@link TLexerSparksql} for tokenization</li>
056 *   <li>Uses {@link TParserSparksql} for parsing</li>
057 *   <li>Tokenization is simple, similar to MySQL</li>
058 *   <li>Delimiter character: ';' for SparkSQL statements</li>
059 * </ul>
060 *
061 * @see SqlParser
062 * @see AbstractSqlParser
063 * @see TLexerSparksql
064 * @see TParserSparksql
065 * @since 3.2.0.0
066 */
067public class SparksqlSqlParser extends AbstractSqlParser {
068
069    /**
070     * Construct SparkSQL parser.
071     * <p>
072     * Configures the parser for Apache Spark SQL with default delimiter (;).
073     */
074    public SparksqlSqlParser() {
075        super(EDbVendor.dbvsparksql);
076        this.delimiterChar = ';';
077        this.defaultDelimiterStr = ";";
078
079        // Create lexer once - will be reused for all parsing operations
080        this.flexer = new TLexerSparksql();
081        this.flexer.delimiterchar = this.delimiterChar;
082        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;
083
084        // Set parent's lexer reference for shared tokenization logic
085        this.lexer = this.flexer;
086
087        // Create parser once - will be reused for all parsing operations
088        this.fparser = new TParserSparksql(null);
089        this.fparser.lexer = this.flexer;
090    }
091
092    // ========== Parser Components ==========
093
094    /** The SparkSQL lexer used for tokenization */
095    public TLexerSparksql flexer;
096
097    /** SparkSQL parser (for SparkSQL statements) */
098    private TParserSparksql fparser;
099
100    /** Current statement being built during extraction */
101    private TCustomSqlStatement gcurrentsqlstatement;
102
103    /** User-defined delimiter string */
104    private String userDelimiterStr = ";";
105
106    /** Current delimiter character */
107    private char curdelimiterchar = ';';
108
109    // ========== AbstractSqlParser Abstract Methods Implementation ==========
110
111    /**
112     * Return the SparkSQL lexer instance.
113     */
114    @Override
115    protected TCustomLexer getLexer(ParserContext context) {
116        return this.flexer;
117    }
118
119    /**
120     * Return the SparkSQL parser instance with updated token list.
121     */
122    @Override
123    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
124        this.fparser.sourcetokenlist = tokens;
125        return this.fparser;
126    }
127
128    /**
129     * Call SparkSQL-specific tokenization logic.
130     */
131    @Override
132    protected void tokenizeVendorSql() {
133        dosparksqltexttotokenlist();
134    }
135
136    /**
137     * Setup SparkSQL parser for raw statement extraction.
138     */
139    @Override
140    protected void setupVendorParsersForExtraction() {
141        this.fparser.sqlcmds = this.sqlcmds;
142        this.fparser.sourcetokenlist = this.sourcetokenlist;
143    }
144
145    /**
146     * Call SparkSQL-specific raw statement extraction logic.
147     */
148    @Override
149    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
150        dosparksqlgetrawsqlstatements(builder);
151    }
152
153    /**
154     * Perform full parsing of statements with syntax checking.
155     */
156    @Override
157    protected TStatementList performParsing(ParserContext context,
158                                           TCustomParser parser,
159                                           TCustomParser secondaryParser,
160                                           TSourceTokenList tokens,
161                                           TStatementList rawStatements) {
162        this.fparser = (TParserSparksql) parser;
163        this.sourcetokenlist = tokens;
164        this.parserContext = context;
165        this.sqlstatements = rawStatements;
166
167        this.sqlcmds = SqlCmdsFactory.get(vendor);
168        this.fparser.sqlcmds = this.sqlcmds;
169
170        if (context != null && context.getGsqlparser() != null) {
171            TGSqlParser gsqlparser = (TGSqlParser) context.getGsqlparser();
172            this.frameStack = gsqlparser.getFrameStack();
173            this.fparser.getNf().setGsqlParser(gsqlparser);
174            this.globalContext = new TContext();
175            this.sqlEnv = new TSQLEnv(this.vendor) {
176                @Override
177                public void initSQLEnv() {
178                }
179            };
180            this.globalContext.setSqlEnv(this.sqlEnv, this.sqlstatements);
181        } else {
182            initializeGlobalContext();
183        }
184
185        for (int i = 0; i < sqlstatements.size(); i++) {
186            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);
187
188            try {
189                stmt.setFrameStack(frameStack);
190                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());
191
192                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
193                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
194                    handleCreateTableErrorRecovery(stmt);
195                }
196
197                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
198                    copyErrorsFromStatement(stmt);
199                }
200
201            } catch (Exception ex) {
202                handleStatementParsingException(stmt, i, ex);
203                continue;
204            }
205        }
206
207        if (globalFrame != null) {
208            globalFrame.popMeFromStack(frameStack);
209        }
210
211        return this.sqlstatements;
212    }
213
214    private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
215        if (((stmt.sqlstatementtype == ESqlStatementType.sstcreatetable)
216                || (stmt.sqlstatementtype == ESqlStatementType.sstcreateindex))
217                && (!TBaseType.c_createTableStrictParsing)) {
218
219            int nested = 0;
220            boolean isIgnore = false, isFoundIgnoreToken = false;
221            TSourceToken firstIgnoreToken = null;
222
223            for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
224                TSourceToken st = stmt.sourcetokenlist.get(k);
225                if (isIgnore) {
226                    if (st.issolidtoken() && (st.tokencode != ';')) {
227                        isFoundIgnoreToken = true;
228                        if (firstIgnoreToken == null) {
229                            firstIgnoreToken = st;
230                        }
231                    }
232                    if (st.tokencode != ';') {
233                        st.tokencode = TBaseType.sqlpluscmd;
234                    }
235                    continue;
236                }
237                if (st.tokencode == (int) ')') {
238                    nested--;
239                    if (nested == 0) {
240                        boolean isSelect = false;
241                        TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
242                        if (st1 != null) {
243                            TSourceToken st2 = st.searchToken((int) '(', 2);
244                            if (st2 != null) {
245                                TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
246                                isSelect = (st3 != null);
247                            }
248                        }
249                        if (!isSelect) isIgnore = true;
250                    }
251                } else if (st.tokencode == (int) '(') {
252                    nested++;
253                }
254            }
255
256            if (isFoundIgnoreToken) {
257                stmt.clearError();
258                stmt.parsestatement(null, false);
259            }
260        }
261    }
262
263    @Override
264    protected void performSemanticAnalysis(ParserContext context, TStatementList statements) {
265        if (context != null && context.getGsqlparser() != null) {
266            return;
267        }
268
269        if (getSyntaxErrors().isEmpty()) {
270            if (TBaseType.isEnableResolver2()) {
271                TSQLResolverConfig config = new TSQLResolverConfig();
272                config.setVendor(vendor);
273                TSQLResolver2 resolver2 = new TSQLResolver2(null, statements, config);
274                if (this.sqlEnv != null) {
275                    resolver2.setSqlEnv(this.sqlEnv);
276                }
277                resolver2.resolve();
278            } else if (TBaseType.isEnableResolver()) {
279                TSQLResolver resolver = new TSQLResolver(globalContext, statements);
280                resolver.resolve();
281            }
282        }
283    }
284
285    @Override
286    protected void performInterpreter(ParserContext context, TStatementList statements) {
287        if (TBaseType.ENABLE_INTERPRETER && getSyntaxErrors().isEmpty()) {
288            TLog.clearLogs();
289            TGlobalScope interpreterScope = new TGlobalScope(sqlEnv);
290            TLog.enableInterpreterLogOnly();
291            TASTEvaluator astEvaluator = new TASTEvaluator(statements, interpreterScope);
292            astEvaluator.eval();
293        }
294    }
295
296    // ========== Helper Methods ==========
297
298    /**
299     * Add token to current statement with proper statement linkage.
300     * This replicates TCustomSqlStatement.addtokentolist() behavior
301     * which sets st.stmt = this before adding.
302     */
303    private void addTokenToStatement(TSourceToken st) {
304        st.stmt = gcurrentsqlstatement;
305        gcurrentsqlstatement.sourcetokenlist.add(st);
306    }
307
308    // ========== SparkSQL-Specific Tokenization ==========
309
310    /**
311     * SparkSQL-specific tokenization logic.
312     * <p>
313     * SparkSQL uses simple tokenization, similar to MySQL.
314     * Handles ROLLUP keyword specially.
315     * Migrated from TGSqlParser.dosparksqltexttotokenlist()
316     */
317    private void dosparksqltexttotokenlist() {
318        TSourceToken asourcetoken, lcprevst;
319        int yychar;
320        boolean startDelimiter = false;
321
322        flexer.tmpDelimiter = "";
323
324        asourcetoken = getanewsourcetoken();
325        if (asourcetoken == null) return;
326        yychar = asourcetoken.tokencode;
327
328        while (yychar > 0) {
329            // Always split mysqllabel ("ident:") into IDENT + ':' tokens.
330            // The grammar handles labeled blocks via IDENT ':' BEGIN.
331            if (yychar == TBaseType.mysqllabel) {
332                String labelText = asourcetoken.toString();
333                String identText = labelText.substring(0, labelText.length() - 1);
334                int line = (int) asourcetoken.lineNo;
335                int col = (int) asourcetoken.columnNo;
336
337                asourcetoken.setAstext(identText);
338                asourcetoken.tokencode = TBaseType.ident;
339                asourcetoken.tokentype = ETokenType.ttidentifier;
340                sourcetokenlist.add(asourcetoken);
341
342                TSourceToken colonToken = new TSourceToken(":");
343                colonToken.tokencode = ':';
344                colonToken.tokentype = ETokenType.ttcolon;
345                colonToken.tokenstatus = ETokenStatus.tsoriginal;
346                colonToken.lineNo = line;
347                colonToken.columnNo = col + identText.length();
348                colonToken.container = sourcetokenlist;
349                sourcetokenlist.curpos = sourcetokenlist.curpos + 1;
350                colonToken.posinlist = sourcetokenlist.curpos;
351                sourcetokenlist.add(colonToken);
352
353                asourcetoken = getanewsourcetoken();
354                if (asourcetoken == null) break;
355                yychar = asourcetoken.tokencode;
356                continue;
357            }
358
359            sourcetokenlist.add(asourcetoken);
360            asourcetoken = getanewsourcetoken();
361            if (asourcetoken == null) break;
362            checkMySQLCommentToken(asourcetoken);
363
364            if ((asourcetoken.tokencode == TBaseType.lexnewline) && (startDelimiter)) {
365                startDelimiter = false;
366                flexer.tmpDelimiter = sourcetokenlist.get(sourcetokenlist.size() - 1).getAstext();
367            }
368
369            if (asourcetoken.tokencode == TBaseType.rrw_rollup) {
370                // with rollup
371                lcprevst = getprevsolidtoken(asourcetoken);
372                if (lcprevst != null) {
373                    if (lcprevst.tokencode == TBaseType.rrw_with)
374                        lcprevst.tokencode = TBaseType.with_rollup;
375                }
376            }
377
378            yychar = asourcetoken.tokencode;
379        }
380    }
381
382    private TSourceToken getprevsolidtoken(TSourceToken ptoken) {
383        if (ptoken == null) return null;
384        TSourceTokenList lcstlist = ptoken.container;
385        if (TBaseType.assigned(lcstlist)) {
386            return lcstlist.nextsolidtoken(ptoken.posinlist - 1, -1, false);
387        }
388        return null;
389    }
390
391    /**
392     * Check and process MySQL-style comment tokens.
393     * <p>
394     * Migrated from TGSqlParser.checkMySQLCommentToken()
395     */
396    private void checkMySQLCommentToken(TSourceToken asourcetoken) {
397        if (asourcetoken.tokencode == TBaseType.cmtslashstar) {
398            String cmtText = asourcetoken.toString();
399            if (cmtText.startsWith("/*+") || cmtText.startsWith("/*-") || cmtText.startsWith("/*!")
400                    || cmtText.startsWith("/*%")) {
401                // Optimizer hint comment, leave as is
402            }
403        }
404    }
405
406    // ========== SparkSQL-Specific Raw Statement Extraction ==========
407
408    /**
409     * SparkSQL-specific raw statement extraction logic.
410     * <p>
411     * SparkSQL uses MySQL-like raw statement extraction with special handling
412     * for DATE, TIME, TIMESTAMP, and INTERVAL keywords.
413     * Migrated from TGSqlParser.dosparksqlgetrawsqlstatements()
414     */
415    private void dosparksqlgetrawsqlstatements(SqlParseResult.Builder builder) {
416        int errorcount = 0;
417        gcurrentsqlstatement = null;
418        EFindSqlStateType gst = EFindSqlStateType.stnormal;
419        int i;
420        TSourceToken ast;
421        boolean waitingDelimiter = false;
422        int compoundBlockNesting = 0;
423
424        // reset delimiter
425        userDelimiterStr = defaultDelimiterStr;
426
427        for (i = 0; i < sourcetokenlist.size(); i++) {
428            ast = sourcetokenlist.get(i);
429            sourcetokenlist.curpos = i;
430
431            // Handle SparkSQL-specific keywords
432            if (ast.tokencode == TBaseType.rrw_date) {
433                TSourceToken st1 = ast.nextSolidToken();
434                if (st1 != null) {
435                    if (st1.tokencode == '(') {
436                        ast.tokencode = TBaseType.rrw_spark_date_function;
437                    } else if (st1.tokencode == TBaseType.sconst) {
438                        ast.tokencode = TBaseType.rrw_spark_date_const;
439                    }
440                }
441            } else if (ast.tokencode == TBaseType.rrw_time) {
442                TSourceToken st1 = ast.nextSolidToken();
443                if (st1 != null) {
444                    if (st1.tokencode == TBaseType.sconst) {
445                        ast.tokencode = TBaseType.rrw_spark_time_const;
446                    }
447                }
448            } else if (ast.tokencode == TBaseType.rrw_timestamp) {
449                TSourceToken st1 = ast.nextSolidToken();
450                if (st1 != null) {
451                    if (st1.tokencode == TBaseType.sconst) {
452                        ast.tokencode = TBaseType.rrw_spark_timestamp_constant;
453                    } else if (st1.tokencode == TBaseType.ident) {
454                        if (st1.toString().startsWith("\"")) {
455                            ast.tokencode = TBaseType.rrw_spark_timestamp_constant;
456                            st1.tokencode = TBaseType.sconst;
457                        }
458                    }
459                }
460            } else if (ast.tokencode == TBaseType.rrw_interval) {
461                TSourceToken leftParen = ast.searchToken('(', 1);
462                if (leftParen != null) {
463                    int k = leftParen.posinlist + 1;
464                    boolean commaToken = false;
465                    while (k < ast.container.size()) {
466                        if (ast.container.get(k).tokencode == ')') break;
467                        if (ast.container.get(k).tokencode == ',') {
468                            commaToken = true;
469                            break;
470                        }
471                        k++;
472                    }
473                    if (commaToken) {
474                        ast.tokencode = TBaseType.rrw_mysql_interval_func;
475                    }
476                }
477            } else if (ast.tokencode == TBaseType.rrw_spark_position) {
478                TSourceToken leftParen = ast.searchToken('(', 1);
479                if (leftParen == null) {
480                    ast.tokencode = TBaseType.ident; // treat it as identifier
481                }
482            }
483
484            switch (gst) {
485                case sterror: {
486                    if (ast.tokentype == ETokenType.ttsemicolon) {
487                        addTokenToStatement(ast);
488                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
489                        gst = EFindSqlStateType.stnormal;
490                    } else {
491                        addTokenToStatement(ast);
492                    }
493                    break;
494                }
495                case stnormal: {
496                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
497                            || (ast.tokencode == TBaseType.cmtslashstar)
498                            || (ast.tokencode == TBaseType.lexspace)
499                            || (ast.tokencode == TBaseType.lexnewline)
500                            || (ast.tokentype == ETokenType.ttsemicolon)) {
501                        if (TBaseType.assigned(gcurrentsqlstatement)) {
502                            addTokenToStatement(ast);
503                        }
504
505                        continue;
506                    }
507
508                    if ((ast.isFirstTokenOfLine()) && ((ast.tokencode == TBaseType.rrw_mysql_source) || (ast.tokencode == TBaseType.slash_dot))) {
509                        gst = EFindSqlStateType.stsqlplus;
510                        gcurrentsqlstatement = new TMySQLSource(vendor);
511                        addTokenToStatement(ast);
512                        continue;
513                    }
514
515                    // find a tokentext to start sql or plsql mode
516                    gcurrentsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);
517
518                    if (TBaseType.assigned(gcurrentsqlstatement)) {
519                        ESqlStatementType[] ses = {ESqlStatementType.sstmysqlcreateprocedure, ESqlStatementType.sstmysqlcreatefunction
520                                , sstcreateprocedure, ESqlStatementType.sstcreatefunction
521                                , ESqlStatementType.sstcreatetrigger};
522                        if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sst_plsql_block) {
523                            // Compound BEGIN...END block
524                            gst = EFindSqlStateType.stsql;
525                            compoundBlockNesting = (ast.tokencode == TBaseType.rrw_begin) ? 1 : 0;
526                            addTokenToStatement(ast);
527                        } else if (includesqlstatementtype(gcurrentsqlstatement.sqlstatementtype, ses)) {
528                            gst = EFindSqlStateType.ststoredprocedure;
529                            waitingDelimiter = false;
530                            addTokenToStatement(ast);
531                            curdelimiterchar = ';';
532                        } else {
533                            gst = EFindSqlStateType.stsql;
534                            addTokenToStatement(ast);
535                        }
536
537                    }
538
539                    if (!TBaseType.assigned(gcurrentsqlstatement)) { // error tokentext found
540
541                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo, (ast.columnNo < 0 ? 0 : ast.columnNo)
542                                , "Error when tokenlize", EErrorType.spwarning, TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));
543
544                        ast.tokentype = ETokenType.tttokenlizererrortoken;
545                        gst = EFindSqlStateType.sterror;
546
547                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
548                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
549                        addTokenToStatement(ast);
550
551                    }
552                    break;
553                }
554                case stsqlplus: {
555                    if (ast.tokencode == TBaseType.lexnewline) {
556                        gst = EFindSqlStateType.stnormal;
557                        addTokenToStatement(ast); // so add it here
558                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
559                    } else {
560                        addTokenToStatement(ast);
561                    }
562
563                    break;
564                }
565                case stsql: {
566                    // Track BEGIN/END nesting for compound blocks
567                    if (gcurrentsqlstatement != null
568                            && gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sst_plsql_block) {
569                        if (ast.tokencode == TBaseType.rrw_begin) {
570                            compoundBlockNesting++;
571                        } else if (ast.tokencode == TBaseType.rrw_end && compoundBlockNesting > 0) {
572                            // Only decrement for END that closes a BEGIN block.
573                            // END IF, END WHILE, END LOOP, END REPEAT, END CASE don't close BEGIN.
574                            TSourceToken nextSolid = sourcetokenlist.nextsolidtoken(i, 1, false);
575                            boolean isStructEnd = nextSolid != null && (
576                                nextSolid.tokencode == TBaseType.rrw_if ||
577                                nextSolid.tokencode == TBaseType.rrw_while ||
578                                nextSolid.tokencode == TBaseType.rrw_loop ||
579                                nextSolid.tokencode == TBaseType.rrw_repeat ||
580                                nextSolid.tokencode == TBaseType.rrw_case ||
581                                nextSolid.tokencode == TBaseType.rrw_for);
582                            if (!isStructEnd) {
583                                compoundBlockNesting--;
584                            }
585                        }
586                    }
587
588                    if ((ast.tokentype == ETokenType.ttsemicolon) && (gcurrentsqlstatement.sqlstatementtype != ESqlStatementType.sstmysqldelimiter)) {
589                        if (compoundBlockNesting > 0) {
590                            // Inside compound block - don't complete on semicolon
591                            addTokenToStatement(ast);
592                            continue;
593                        }
594                        gst = EFindSqlStateType.stnormal;
595                        addTokenToStatement(ast);
596                        gcurrentsqlstatement.semicolonended = ast;
597                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
598                        compoundBlockNesting = 0;
599                        continue;
600                    }
601                    if (ast.toString().equalsIgnoreCase(userDelimiterStr)) {
602                        gst = EFindSqlStateType.stnormal;
603                        ast.tokencode = ';'; // treat it as semicolon
604                        addTokenToStatement(ast);
605                        gcurrentsqlstatement.semicolonended = ast;
606                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
607                        continue;
608                    }
609                    addTokenToStatement(ast);
610
611                    if ((ast.tokencode == TBaseType.lexnewline)
612                            && (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmysqldelimiter)) {
613                        gst = EFindSqlStateType.stnormal;
614                        userDelimiterStr = "";
615                        for (int k = 0; k < gcurrentsqlstatement.sourcetokenlist.size(); k++) {
616                            TSourceToken st = gcurrentsqlstatement.sourcetokenlist.get(k);
617                            if ((st.tokencode == TBaseType.rrw_mysql_delimiter)
618                                    || (st.tokencode == TBaseType.lexnewline)
619                                    || (st.tokencode == TBaseType.lexspace)
620                                    || (st.tokencode == TBaseType.rrw_set)) {
621                                continue;
622                            }
623
624                            userDelimiterStr += st.toString();
625                        }
626                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
627
628                        continue;
629                    }
630
631                    break;
632                }
633                case ststoredprocedure: {
634                    if (waitingDelimiter) {
635                        if (userDelimiterStr.equalsIgnoreCase(ast.toString())) {
636                            gst = EFindSqlStateType.stnormal;
637                            gcurrentsqlstatement.semicolonended = ast;
638                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
639                            continue;
640                        } else if (userDelimiterStr.startsWith(ast.toString())) {
641                            String lcstr = ast.toString();
642                            for (int k = ast.posinlist + 1; k < ast.container.size(); k++) {
643                                TSourceToken st = ast.container.get(k);
644                                if ((st.tokencode == TBaseType.rrw_mysql_delimiter) || (st.tokencode == TBaseType.lexnewline) || (st.tokencode == TBaseType.lexspace)) {
645                                    break;
646                                }
647                                lcstr = lcstr + st.toString();
648                            }
649
650                            if (userDelimiterStr.equalsIgnoreCase(lcstr)) {
651                                for (int k = ast.posinlist; k < ast.container.size(); k++) {
652                                    TSourceToken st = ast.container.get(k);
653                                    if ((st.tokencode == TBaseType.rrw_mysql_delimiter) || (st.tokencode == TBaseType.lexnewline) || (st.tokencode == TBaseType.lexspace)) {
654                                        break;
655                                    }
656                                    ast.tokenstatus = ETokenStatus.tsignorebyyacc;
657                                }
658                                gst = EFindSqlStateType.stnormal;
659                                gcurrentsqlstatement.semicolonended = ast;
660                                onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
661                                continue;
662                            }
663
664                        }
665                    }
666                    if (ast.tokencode == TBaseType.rrw_begin)
667                        waitingDelimiter = true;
668
669                    if (userDelimiterStr.equals(";") || (waitingDelimiter == false)) {
670                        addTokenToStatement(ast);
671                        if (ast.tokentype == ETokenType.ttsemicolon) {
672                            gst = EFindSqlStateType.stnormal;
673                            gcurrentsqlstatement.semicolonended = ast;
674                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
675                            continue;
676                        }
677                    } else {
678                        if (ast.toString().equals(userDelimiterStr)) {
679                            ast.tokenstatus = ETokenStatus.tsignorebyyacc;
680                            addTokenToStatement(ast);
681                            gst = EFindSqlStateType.stnormal;
682                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
683                        } else {
684                            if ((ast.tokentype == ETokenType.ttsemicolon) && (userDelimiterStr.equals(";"))) {
685                                TSourceToken lcprevtoken = ast.container.nextsolidtoken(ast, -1, false);
686                                if (lcprevtoken != null) {
687                                    if (lcprevtoken.tokencode == TBaseType.rrw_end) {
688                                        gst = EFindSqlStateType.stnormal;
689                                        gcurrentsqlstatement.semicolonended = ast;
690                                        addTokenToStatement(ast);
691                                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
692                                        continue;
693                                    }
694                                }
695                            }
696
697                            addTokenToStatement(ast);
698                        }
699                    }
700                    break;
701                }
702            }
703        }
704
705        // last statement
706        if (TBaseType.assigned(gcurrentsqlstatement) && ((gst == EFindSqlStateType.stsql) || (gst == EFindSqlStateType.ststoredprocedure) || (gst == EFindSqlStateType.sterror))) {
707            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, true, builder);
708        }
709
710        builder.sqlStatements(this.sqlstatements);
711        builder.errorCode(errorcount);
712        builder.errorMessage(errorcount == 0 ? "" : String.format("Extraction completed with %d error(s)", errorcount));
713    }
714
715    /**
716     * Check if a SQL statement type is included in an array.
717     */
718    private boolean includesqlstatementtype(ESqlStatementType type, ESqlStatementType[] types) {
719        for (ESqlStatementType t : types) {
720            if (t == type) return true;
721        }
722        return false;
723    }
724}