Source code

001package gudusoft.gsqlparser.parser;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.TBaseType;
005import gudusoft.gsqlparser.TGSqlParser;
006import gudusoft.gsqlparser.TCustomLexer;
007import gudusoft.gsqlparser.TCustomParser;
008import gudusoft.gsqlparser.TCustomSqlStatement;
009import gudusoft.gsqlparser.TLexerSybaseiq;
010import gudusoft.gsqlparser.TParserSybaseiq;
011import gudusoft.gsqlparser.TSourceToken;
012import gudusoft.gsqlparser.TSourceTokenList;
013import gudusoft.gsqlparser.TStatementList;
014import gudusoft.gsqlparser.TSyntaxError;
015import gudusoft.gsqlparser.EFindSqlStateType;
016import gudusoft.gsqlparser.ETokenType;
017import gudusoft.gsqlparser.ETokenStatus;
018import gudusoft.gsqlparser.ESqlStatementType;
019import gudusoft.gsqlparser.EErrorType;
020import gudusoft.gsqlparser.stmt.TUnknownSqlStatement;
021import gudusoft.gsqlparser.stmt.mssql.TMssqlBlock;
022import gudusoft.gsqlparser.stmt.mssql.TMssqlExecute;
023import gudusoft.gsqlparser.sqlcmds.ISqlCmds;
024import gudusoft.gsqlparser.sqlcmds.SqlCmdsFactory;
025import gudusoft.gsqlparser.compiler.TContext;
026import gudusoft.gsqlparser.sqlenv.TSQLEnv;
027import gudusoft.gsqlparser.compiler.TGlobalScope;
028import gudusoft.gsqlparser.compiler.TFrame;
029import gudusoft.gsqlparser.resolver.TSQLResolver;
030import gudusoft.gsqlparser.resolver2.TSQLResolver2;
031import gudusoft.gsqlparser.resolver2.TSQLResolverConfig;
032import gudusoft.gsqlparser.TLog;
033import gudusoft.gsqlparser.compiler.TASTEvaluator;
034
035import java.io.BufferedReader;
036import java.util.ArrayList;
037import java.util.List;
038import java.util.Stack;
039
040/**
041 * Sybase IQ SQL parser implementation.
042 *
043 * <p>This parser handles Sybase IQ-specific SQL syntax including:
044 * <ul>
045 *   <li>Sybase stored procedures with GO statement boundaries</li>
046 *   <li>Sybase BEGIN/END blocks</li>
047 *   <li>Sybase TRY/CATCH blocks</li>
048 *   <li>Sybase IQ-specific statement types</li>
049 * </ul>
050 *
051 * <p><b>Design Notes:</b>
052 * <ul>
053 *   <li>Extends {@link AbstractSqlParser} using the template method pattern</li>
054 *   <li>Uses {@link TLexerSybaseiq} for tokenization</li>
055 *   <li>Uses {@link TParserSybaseiq} for parsing</li>
056 *   <li>Tokenization and statement extraction similar to MSSQL</li>
057 *   <li>Delimiter character: ';' for Sybase statements</li>
058 * </ul>
059 *
060 * @see SqlParser
061 * @see AbstractSqlParser
062 * @see TLexerSybaseiq
063 * @see TParserSybaseiq
064 * @since 3.2.0.0
065 */
066public class SybaseIqSqlParser extends AbstractSqlParser {
067
068    /**
069     * Policy layer of the recognize-then-reject design (see
070     * docs/designs/sybase-tsql-vendor-decomposition.md).
071     *
072     * <p>The splitter registers the FULL family command set, so statement
073     * boundaries are always correct even in multi-statement input and in both
074     * splitter modes. Statements recognised as types this product does not
075     * have are then rejected HERE, with the statement type preserved and the
076     * error anchored at the statement's first token.
077     *
078     * <p>De-registration was the previous mechanism and is retired: it made
079     * the splitter mis-attach forbidden statements to their predecessor in
080     * multi-statement input (swallowing them with rc=0), and crashed outright
081     * where the grammar builds a dedicated node.
082     */
083    private static final java.util.EnumSet<ESqlStatementType> FORBIDDEN_STATEMENTS =
084        java.util.EnumSet.of(
085            ESqlStatementType.sstsybasedumpTran,
086            ESqlStatementType.sstmssqlreadtext,
087            ESqlStatementType.sstsybaseupdateindexstatistics,
088            ESqlStatementType.sstsybaseDeleteStatistics);
089
090    private static final java.util.Map<ESqlStatementType, String> FORBIDDEN_MESSAGES =
091        new java.util.EnumMap<>(ESqlStatementType.class);
092    static {
093        FORBIDDEN_MESSAGES.put(ESqlStatementType.sstsybasedumpTran,
094                "DUMP TRANSACTION is Sybase ASE syntax; use BACKUP and is not valid for Sybase IQ");
095        FORBIDDEN_MESSAGES.put(ESqlStatementType.sstmssqlreadtext,
096                "READTEXT is Sybase ASE syntax and is not valid for Sybase IQ");
097        FORBIDDEN_MESSAGES.put(ESqlStatementType.sstsybaseupdateindexstatistics,
098                "UPDATE INDEX/ALL/TABLE STATISTICS is Sybase ASE syntax and is not valid for Sybase IQ");
099        FORBIDDEN_MESSAGES.put(ESqlStatementType.sstsybaseDeleteStatistics,
100                "DELETE STATISTICS is Sybase ASE syntax; use DROP STATISTICS and is not valid for Sybase IQ");
101    }
102
103    /**
104     * Post-parse policy validation ("recognize then reject"). Called by
105     * TGSqlParser after every statement is parsed. Attaches a vendor error to
106     * each recognised-but-foreign statement and returns them.
107     *
108     * <p>Not hooked at raw-extraction time: parsing REPLACES some statement
109     * objects (the refined statement is rebuilt from the parse-tree node — e.g.
110     * DELETE STATISTICS and READTEXT), and an error attached to the raw object
111     * vanishes with it. Post-parse, the statement list is final. This was
112     * measured, not assumed: raw-attached errors survived for DUMP TRAN but
113     * disappeared for DELETE STATISTICS and READTEXT.
114     */
115    @Override
116    public java.util.List<TCustomSqlStatement> validateRecognizedStatements(TStatementList stmts) {
117        java.util.List<TCustomSqlStatement> rejected = new java.util.ArrayList<>();
118        if (stmts == null) return rejected;
119        for (int i = 0; i < stmts.size(); i++) {
120            TCustomSqlStatement st = stmts.get(i);
121            if (st == null || !FORBIDDEN_STATEMENTS.contains(st.sqlstatementtype)) continue;
122            rejected.add(st);
123            TSourceToken tok = (st.sourcetokenlist != null && st.sourcetokenlist.size() > 0)
124                    ? st.sourcetokenlist.get(0) : null;
125            String msg = FORBIDDEN_MESSAGES.get(st.sqlstatementtype);
126            if (tok != null) {
127                st.parseerrormessagehandle(new TSyntaxError(tok.toString(), tok.lineNo,
128                        tok.columnNo, msg, EErrorType.sperror, 10101, st, tok.posinlist));
129            } else {
130                st.parseerrormessagehandle(new TSyntaxError(msg, 0, 0, msg,
131                        EErrorType.sperror, 10101, st, -1));
132            }
133        }
134        return rejected;
135    }
136
137
138    /**
139     * Construct Sybase IQ SQL parser.
140     * <p>
141     * Configures the parser for Sybase with default delimiter (;).
142     */
143    public SybaseIqSqlParser() {
144        super(EDbVendor.dbvsybaseiq);
145        this.delimiterChar = ';';
146        this.defaultDelimiterStr = ";";
147
148        // Create lexer once - will be reused for all parsing operations
149        this.flexer = new TLexerSybaseiq();
150        this.flexer.delimiterchar = this.delimiterChar;
151        this.flexer.defaultDelimiterStr = this.defaultDelimiterStr;
152
153        // Set parent's lexer reference for shared tokenization logic
154        this.lexer = this.flexer;
155
156        // Create parser once - will be reused for all parsing operations
157        this.fparser = new TParserSybaseiq(null);
158        this.fparser.lexer = this.flexer;
159    }
160
161    // ========== Parser Components ==========
162
163    /** The Sybase lexer used for tokenization */
164    public TLexerSybaseiq flexer;
165
166    /** Sybase parser (for Sybase statements) */
167    private TParserSybaseiq fparser;
168
169    /** Current statement being built during extraction */
170    private TCustomSqlStatement gcurrentsqlstatement;
171
172    // ========== AbstractSqlParser Abstract Methods Implementation ==========
173
174    /**
175     * Return the Sybase lexer instance.
176     */
177    @Override
178    protected TCustomLexer getLexer(ParserContext context) {
179        return this.flexer;
180    }
181
182    /**
183     * Return the Sybase parser instance with updated token list.
184     */
185    @Override
186    protected TCustomParser getParser(ParserContext context, TSourceTokenList tokens) {
187        this.fparser.sourcetokenlist = tokens;
188        return this.fparser;
189    }
190
191    /**
192     * Call Sybase IQ-specific tokenization logic.
193     * <p>
194     * Sybase uses MSSQL-like tokenization.
195     */
196    @Override
197    protected void tokenizeVendorSql() {
198        dosybasesqltexttotokenlist();
199    }
200
201    /**
202     * Setup Sybase parser for raw statement extraction.
203     */
204    @Override
205    protected void setupVendorParsersForExtraction() {
206        this.fparser.sqlcmds = this.sqlcmds;
207        this.fparser.sourcetokenlist = this.sourcetokenlist;
208    }
209
210    /**
211     * Call Sybase IQ-specific raw statement extraction logic.
212     */
213    @Override
214    protected void extractVendorRawStatements(SqlParseResult.Builder builder) {
215        dosybasegetrawsqlstatements(builder);
216    }
217
218    /**
219     * Perform full parsing of statements with syntax checking.
220     */
221    @Override
222    protected TStatementList performParsing(ParserContext context,
223                                           TCustomParser parser,
224                                           TCustomParser secondaryParser,
225                                           TSourceTokenList tokens,
226                                           TStatementList rawStatements) {
227        this.fparser = (TParserSybaseiq) parser;
228        this.sourcetokenlist = tokens;
229        this.parserContext = context;
230        this.sqlstatements = rawStatements;
231
232        this.sqlcmds = SqlCmdsFactory.get(vendor);
233        this.fparser.sqlcmds = this.sqlcmds;
234
235        if (context != null && context.getGsqlparser() != null) {
236            TGSqlParser gsqlparser = (TGSqlParser) context.getGsqlparser();
237            this.frameStack = gsqlparser.getFrameStack();
238            this.fparser.getNf().setGsqlParser(gsqlparser);
239            this.globalContext = new TContext();
240            this.sqlEnv = new TSQLEnv(this.vendor) {
241                @Override
242                public void initSQLEnv() {
243                }
244            };
245            this.globalContext.setSqlEnv(this.sqlEnv, this.sqlstatements);
246        } else {
247            initializeGlobalContext();
248        }
249
250        for (int i = 0; i < sqlstatements.size(); i++) {
251            TCustomSqlStatement stmt = sqlstatements.getRawSql(i);
252
253            try {
254                stmt.setFrameStack(frameStack);
255                int parseResult = stmt.parsestatement(null, false, context.isOnlyNeedRawParseTree());
256
257                boolean doRecover = TBaseType.ENABLE_ERROR_RECOVER_IN_CREATE_TABLE;
258                if (doRecover && ((parseResult != 0) || (stmt.getErrorCount() > 0))) {
259                    handleCreateTableErrorRecovery(stmt);
260                }
261
262                if ((parseResult != 0) || (stmt.getErrorCount() > 0)) {
263                    copyErrorsFromStatement(stmt);
264                }
265
266            } catch (Exception ex) {
267                handleStatementParsingException(stmt, i, ex);
268                continue;
269            }
270        }
271
272        if (globalFrame != null) {
273            globalFrame.popMeFromStack(frameStack);
274        }
275
276        return this.sqlstatements;
277    }
278
279    private void handleCreateTableErrorRecovery(TCustomSqlStatement stmt) {
280        if (((stmt.sqlstatementtype == ESqlStatementType.sstcreatetable)
281                || (stmt.sqlstatementtype == ESqlStatementType.sstcreateindex))
282                && (!TBaseType.c_createTableStrictParsing)) {
283
284            int nested = 0;
285            boolean isIgnore = false, isFoundIgnoreToken = false;
286            TSourceToken firstIgnoreToken = null;
287
288            for (int k = 0; k < stmt.sourcetokenlist.size(); k++) {
289                TSourceToken st = stmt.sourcetokenlist.get(k);
290                if (isIgnore) {
291                    if (st.issolidtoken() && (st.tokencode != ';')) {
292                        isFoundIgnoreToken = true;
293                        if (firstIgnoreToken == null) {
294                            firstIgnoreToken = st;
295                        }
296                    }
297                    if (st.tokencode != ';') {
298                        st.tokencode = TBaseType.sqlpluscmd;
299                    }
300                    continue;
301                }
302                if (st.tokencode == (int) ')') {
303                    nested--;
304                    if (nested == 0) {
305                        boolean isSelect = false;
306                        TSourceToken st1 = st.searchToken(TBaseType.rrw_as, 1);
307                        if (st1 != null) {
308                            TSourceToken st2 = st.searchToken((int) '(', 2);
309                            if (st2 != null) {
310                                TSourceToken st3 = st.searchToken(TBaseType.rrw_select, 3);
311                                isSelect = (st3 != null);
312                            }
313                        }
314                        if (!isSelect) isIgnore = true;
315                    }
316                } else if (st.tokencode == (int) '(') {
317                    nested++;
318                }
319            }
320
321            if (isFoundIgnoreToken) {
322                stmt.clearError();
323                stmt.parsestatement(null, false);
324            }
325        }
326    }
327
328    @Override
329    protected void performSemanticAnalysis(ParserContext context, TStatementList statements) {
330        if (context != null && context.getGsqlparser() != null) {
331            return;
332        }
333
334        if (getSyntaxErrors().isEmpty()) {
335            if (TBaseType.isEnableResolver2()) {
336                TSQLResolverConfig config = new TSQLResolverConfig();
337                config.setVendor(vendor);
338                TSQLResolver2 resolver2 = new TSQLResolver2(null, statements, config);
339                if (this.sqlEnv != null) {
340                    resolver2.setSqlEnv(this.sqlEnv);
341                }
342                resolver2.resolve();
343            } else if (TBaseType.isEnableResolver()) {
344                TSQLResolver resolver = new TSQLResolver(globalContext, statements);
345                resolver.resolve();
346            }
347        }
348    }
349
350    @Override
351    protected void performInterpreter(ParserContext context, TStatementList statements) {
352        if (TBaseType.ENABLE_INTERPRETER && getSyntaxErrors().isEmpty()) {
353            TLog.clearLogs();
354            TGlobalScope interpreterScope = new TGlobalScope(sqlEnv);
355            TLog.enableInterpreterLogOnly();
356            TASTEvaluator astEvaluator = new TASTEvaluator(statements, interpreterScope);
357            astEvaluator.eval();
358        }
359    }
360
361    // ========== Helper Methods ==========
362
363    /**
364     * Add token to current statement with proper statement linkage.
365     * This replicates TCustomSqlStatement.addtokentolist() behavior
366     * which sets st.stmt = this before adding.
367     */
368    private void addTokenToStatement(TSourceToken st) {
369        st.stmt = gcurrentsqlstatement;
370        gcurrentsqlstatement.sourcetokenlist.add(st);
371    }
372
373    // ========== Sybase-Specific Tokenization ==========
374
375    /**
376     * Sybase IQ-specific tokenization logic.
377     * <p>
378     * Sybase uses MSSQL-like tokenization.
379     * Migrated from TGSqlParser.dosybasesqltexttotokenlist()
380     */
381    private void dosybasesqltexttotokenlist() {
382        TSourceToken lcprevtoken = null;
383        int lcsteps = 0;
384        TSourceToken asourcetoken, lctoken, lctoken2, lctoken3;
385        int yychar;
386        boolean iskeywordgo;
387
388        asourcetoken = getanewsourcetoken();
389
390        if (asourcetoken == null) return;
391
392        yychar = asourcetoken.tokencode;
393
394        boolean lcinopenrowset = false;
395        int lcnested = 0;
396
397        while (yychar > 0) {
398
399            if (asourcetoken.tokencode == TBaseType.rrw_openrowset) {
400                lcinopenrowset = true;
401                lcnested = 0;
402            } else if (asourcetoken.tokentype == ETokenType.ttleftparenthesis) {
403                if ((lcsteps > 0) && TBaseType.assigned(lcprevtoken)) {
404                    if (lcprevtoken.tokencode == TBaseType.rrw_primary) {
405                        lcprevtoken.tokencode = TBaseType.rrw_select - 2;
406                        checkconstarinttoken(lcprevtoken);
407                    } else if (lcprevtoken.tokencode == TBaseType.rrw_foreign) {
408                        lcprevtoken.tokencode = TBaseType.rrw_select - 4;
409                        checkconstarinttoken(lcprevtoken);
410                    } else if (lcprevtoken.tokencode == TBaseType.rrw_unique) {
411                        lcprevtoken.tokencode = TBaseType.rrw_select - 1;
412                        checkconstarinttoken(lcprevtoken);
413                    }
414                    lcprevtoken = null;
415                    lcsteps = 0;
416                }
417
418                if (lcinopenrowset)
419                    lcnested++;
420            } else if (asourcetoken.tokentype == ETokenType.ttrightparenthesis) {
421                if (lcinopenrowset) {
422                    if ((lcnested > 0))
423                        lcnested--;
424                    if (lcnested == 0)
425                        lcinopenrowset = false;
426                }
427            } else if (asourcetoken.tokentype == ETokenType.ttsemicolon) {
428                if (lcinopenrowset) {
429                    asourcetoken.tokentype = ETokenType.ttsemicolon2;
430                } else {
431                    lctoken2 = asourcetoken.searchToken(TBaseType.rrw_begin, -1);
432                    if (lctoken2 != null) {
433                        asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
434                        asourcetoken.tokentype = ETokenType.ttsemicolon3;
435                    } else {
436                        lctoken2 = asourcetoken.searchToken(TBaseType.rrw_try, -1);
437                        if (lctoken2 == null) {
438                            lctoken2 = asourcetoken.searchToken(TBaseType.rrw_catch, -1);
439                        }
440                        if (lctoken2 != null) {
441                            lctoken3 = asourcetoken.searchToken(TBaseType.rrw_begin, -2);
442                            if (lctoken3 != null) {
443                                asourcetoken.tokencode = TBaseType.SEMI_COLON_AFTER_BEGIN;
444                                asourcetoken.tokentype = ETokenType.ttsemicolon3;
445                            }
446                        }
447                    }
448                }
449            } else if (asourcetoken.tokentype == ETokenType.ttperiod) {
450                lctoken = getprevtoken(asourcetoken);
451                if (TBaseType.assigned(lctoken)) {
452                    if (lctoken.tokencode == TBaseType.rrw_go) {
453                        lctoken.tokencode = TBaseType.ident;
454                        lctoken.tokentype = ETokenType.ttidentifier;
455                    }
456                }
457            } else if (asourcetoken.tokencode == TBaseType.rrw_table) {
458                lctoken = getprevtoken(asourcetoken);
459                if (TBaseType.assigned(lctoken)) {
460                    if (lctoken.tokencode == TBaseType.rrw_lock) {
461                        lctoken.tokencode = TBaseType.rw_locktable;
462                    }
463                }
464            } else if (asourcetoken.tokencode == TBaseType.rrw_sybase_isolation) {
465                // AT ISOLATION clause - change 'at' token code for grammar recognition
466                lctoken = getprevtoken(asourcetoken);
467                if (TBaseType.assigned(lctoken)) {
468                    if (lctoken.tokencode == TBaseType.rrw_sybase_at) {
469                        lctoken.tokencode = TBaseType.rw_sybase_at1;
470                    }
471                }
472            } else if (asourcetoken.tokencode == TBaseType.rrw_update) {
473                // UPDATE in create trigger context
474                lctoken = getprevtoken(asourcetoken);
475                if (TBaseType.assigned(lctoken)) {
476                    if ((lctoken.tokencode == TBaseType.rrw_not)
477                            || (lctoken.tokencode == TBaseType.rrw_and)
478                            || (lctoken.tokencode == TBaseType.rrw_or)
479                            || (lctoken.tokencode == TBaseType.rrw_if)
480                    ) {
481                        // this is update(column) in create trigger
482                        asourcetoken.tokencode = TBaseType.rw_sybase_update1;
483                    }
484                }
485            } else if (asourcetoken.tokencode == TBaseType.rrw_go) {
486                iskeywordgo = true;
487                lctoken = getprevtoken(asourcetoken);
488                if (TBaseType.assigned(lctoken)) {
489                    if ((lctoken.lineNo == asourcetoken.lineNo) && (lctoken.tokencode != ';')) {
490                        iskeywordgo = false;
491                    }
492                }
493
494                if (iskeywordgo) {
495                    lcinopenrowset = false;
496                    lcnested = 0;
497                    lcprevtoken = asourcetoken;
498                } else {
499                    asourcetoken.tokencode = TBaseType.ident;
500                    asourcetoken.tokentype = ETokenType.ttidentifier;
501                }
502            } else if (asourcetoken.tokencode == TBaseType.rrw_primary) {
503                lcsteps = 2;
504                lcprevtoken = asourcetoken;
505            } else if (asourcetoken.tokencode == TBaseType.rrw_foreign) {
506                lcsteps = 2;
507                lcprevtoken = asourcetoken;
508            } else if (asourcetoken.tokencode == TBaseType.rrw_unique) {
509                lcsteps = 1;
510                lcprevtoken = asourcetoken;
511            } else if (asourcetoken.issolidtoken()) {
512                if (lcsteps > 0) {
513                    if (!(TBaseType.mysametext("clustered", asourcetoken.toString())
514                            || TBaseType.mysametext("nonclustered", asourcetoken.toString())
515                            || TBaseType.mysametext("hash", asourcetoken.toString())
516                    ))
517                        lcsteps--;
518                }
519            }
520
521            sourcetokenlist.add(asourcetoken);
522
523            asourcetoken = getanewsourcetoken();
524            if (asourcetoken == null) break;
525            yychar = asourcetoken.tokencode;
526        }
527    }
528
529    private TSourceToken getprevtoken(TSourceToken ptoken) {
530        TSourceTokenList lcstlist = ptoken.container;
531        if (TBaseType.assigned(lcstlist)) {
532            if ((ptoken.posinlist > 0) && (lcstlist.size() > ptoken.posinlist - 1)) {
533                if (!((lcstlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttwhitespace)
534                        || (lcstlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttreturn)
535                        || (lcstlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttsimplecomment)
536                        || (lcstlist.get(ptoken.posinlist - 1).tokentype == ETokenType.ttbracketedcomment)
537                )) {
538                    return lcstlist.get(ptoken.posinlist - 1);
539                } else {
540                    return lcstlist.nextsolidtoken(ptoken.posinlist - 1, -1, false);
541                }
542            }
543        }
544        return null;
545    }
546
547    private void checkconstarinttoken(TSourceToken token) {
548        TSourceTokenList lcStList = token.container;
549        if (TBaseType.assigned(lcStList)) {
550            TSourceToken lcPPToken = lcStList.nextsolidtoken(token.posinlist, -2, false);
551            if (TBaseType.assigned(lcPPToken)) {
552                if (lcPPToken.tokencode == flexer.getkeywordvalue("constraint")) {
553                    lcPPToken.tokencode = TBaseType.rw_constraint2;
554                }
555            }
556        }
557    }
558
559    // ========== Sybase-Specific Raw Statement Extraction ==========
560
561    /**
562     * Sybase IQ-specific raw statement extraction logic.
563     * <p>
564     * Migrated from TGSqlParser.dosybasegetrawsqlstatements()
565     */
566    private void dosybasegetrawsqlstatements(SqlParseResult.Builder builder) {
567        int errorcount = 0;
568        int case_end_nest = 0;
569        // Watcom / SAP IQ / SQL Anywhere `IF cond THEN ... END IF` tracking.
570        // `lcwatcomIfNest` counts open IF...THEN blocks awaiting END IF or
571        // ENDIF inside a BEGIN/END block. `lcwhenPending` counts WHEN tokens
572        // awaiting their matching THEN so that CASE WHEN ... THEN, MERGE
573        // WHEN ... THEN and EXCEPTION WHEN ... THEN do not look like Watcom
574        // IF openers. The counters are only used in the stblock state — at
575        // top level we still rely on the standard statement-splitter path.
576        int lcwatcomIfNest = 0;
577        int lcwhenPending = 0;
578
579        if (TBaseType.assigned(sqlstatements)) sqlstatements.clear();
580        if (!TBaseType.assigned(sourcetokenlist)) {
581            builder.errorCode(-1);
582            builder.errorMessage("Source token list is null");
583            return;
584        }
585
586        gcurrentsqlstatement = null;
587        EFindSqlStateType gst = EFindSqlStateType.stnormal;
588        int lcblocklevel = 0;
589        int lctrycatchlevel = 0;
590        TSourceToken lcprevsolidtoken = null, lcnextsolidtoken, lcnnextsolidtoken;
591        TSourceToken ast = null;
592        int i;
593        boolean lcisendconversation, lcstillinsql;
594
595        for (i = 0; i < sourcetokenlist.size(); i++) {
596
597            if ((ast != null) && (ast.issolidtoken()))
598                lcprevsolidtoken = ast;
599
600            ast = sourcetokenlist.get(i);
601            sourcetokenlist.curpos = i;
602            if (ast.tokenstatus == ETokenStatus.tsignoredbygetrawstatement) {
603                addTokenToStatement(ast);
604                continue;
605            }
606
607            if (gst == EFindSqlStateType.ststoredprocedurebody) {
608                if (!((ast.tokencode == TBaseType.rrw_go)
609                        || (ast.tokencode == TBaseType.rrw_create)
610                        || (ast.tokencode == TBaseType.rrw_alter))) {
611                    addTokenToStatement(ast);
612                    continue;
613                }
614            }
615
616            TCustomSqlStatement lcnextsqlstatement = sqlcmds.issql(ast, gst, gcurrentsqlstatement);
617
618            switch (gst) {
619                case sterror: {
620                    if (TBaseType.assigned(lcnextsqlstatement)) {
621                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
622                        gcurrentsqlstatement = lcnextsqlstatement;
623                        addTokenToStatement(ast);
624                        gst = EFindSqlStateType.stsql;
625                    } else if ((ast.tokentype == ETokenType.ttsemicolon)) {
626                        addTokenToStatement(ast);
627                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
628                        gst = EFindSqlStateType.stnormal;
629                    } else {
630                        addTokenToStatement(ast);
631                    }
632                    break;
633                }
634                case stnormal: {
635                    if ((ast.tokencode == TBaseType.cmtdoublehyphen)
636                            || (ast.tokencode == TBaseType.cmtslashstar)
637                            || (ast.tokencode == TBaseType.lexspace)
638                            || (ast.tokencode == TBaseType.lexnewline)
639                            || (ast.tokentype == ETokenType.ttsemicolon)) {
640                        if (TBaseType.assigned(gcurrentsqlstatement)) {
641                            addTokenToStatement(ast);
642                        }
643
644                        if (TBaseType.assigned(lcprevsolidtoken) && (ast.tokentype == ETokenType.ttsemicolon)) {
645                            if (lcprevsolidtoken.tokentype == ETokenType.ttsemicolon) {
646                                ast.tokentype = ETokenType.ttsimplecomment;
647                                ast.tokencode = TBaseType.cmtdoublehyphen;
648                            }
649                        }
650
651                        continue;
652                    }
653
654                    gcurrentsqlstatement = lcnextsqlstatement;
655
656                    if (TBaseType.assigned(gcurrentsqlstatement)) {
657                        switch (gcurrentsqlstatement.sqlstatementtype) {
658                            case sstmssqlcreateprocedure:
659                            case sstmssqlcreatefunction:
660                            case sstcreatetrigger:
661                            case sstmssqlalterprocedure:
662                            case sstmssqlalterfunction:
663                            case sstmssqlaltertrigger: {
664                                addTokenToStatement(ast);
665                                gst = EFindSqlStateType.ststoredprocedure;
666                                break;
667                            }
668                            case sstmssqlbegintry:
669                            case sstmssqlbegincatch: {
670                                addTokenToStatement(ast);
671                                gst = EFindSqlStateType.sttrycatch;
672                                lctrycatchlevel = 0;
673                                break;
674                            }
675                            case sstmssqlgo: {
676                                addTokenToStatement(ast);
677                                onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
678                                gst = EFindSqlStateType.stnormal;
679                                break;
680                            }
681                            default: {
682                                addTokenToStatement(ast);
683                                if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqllabel) {
684                                    onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
685                                    gst = EFindSqlStateType.stnormal;
686                                } else {
687                                    gst = EFindSqlStateType.stsql;
688                                }
689                                break;
690                            }
691                        }
692                    } else {
693                        if (ast.tokencode == TBaseType.rrw_begin) {
694                            gcurrentsqlstatement = new TMssqlBlock(vendor);
695                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
696                            addTokenToStatement(ast);
697                            gst = EFindSqlStateType.stblock;
698                        } else {
699                            if (sqlstatements.size() == 0) {
700                                gst = EFindSqlStateType.stsql;
701                                gcurrentsqlstatement = new TMssqlExecute(vendor);
702                                ast.tokencode = TBaseType.rrw_sybase_exce_proc_name;
703                                addTokenToStatement(ast);
704                            } else if (sqlstatements.get(sqlstatements.size() - 1).sqlstatementtype == ESqlStatementType.sstmssqlgo) {
705                                gst = EFindSqlStateType.stsql;
706                                gcurrentsqlstatement = new TMssqlExecute(vendor);
707                                ast.tokencode = TBaseType.rrw_sybase_exce_proc_name;
708                                addTokenToStatement(ast);
709                            }
710                        }
711                    }
712
713                    if (!TBaseType.assigned(gcurrentsqlstatement)) {
714                        this.syntaxErrors.add(new TSyntaxError(ast.getAstext(), ast.lineNo, (ast.columnNo < 0 ? 0 : ast.columnNo)
715                                , "Error when tokenlize", EErrorType.spwarning, TBaseType.MSG_WARNING_ERROR_WHEN_TOKENIZE, null, ast.posinlist));
716
717                        ast.tokentype = ETokenType.tttokenlizererrortoken;
718                        gst = EFindSqlStateType.sterror;
719
720                        gcurrentsqlstatement = new TUnknownSqlStatement(vendor);
721                        gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstinvalid;
722                        addTokenToStatement(ast);
723                    }
724                    break;
725                }
726                case stblock: {
727                    if (TBaseType.assigned(lcnextsqlstatement)) {
728                        if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo) {
729                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
730                            gcurrentsqlstatement = lcnextsqlstatement;
731                            addTokenToStatement(ast);
732                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
733                            gst = EFindSqlStateType.stnormal;
734                        } else {
735                            lcnextsqlstatement = null;
736                        }
737                    }
738
739                    if (gst == EFindSqlStateType.stblock) {
740                        addTokenToStatement(ast);
741                        if (ast.tokencode == TBaseType.rrw_begin) {
742                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i, 1, false);
743                            if (TBaseType.assigned(lcnextsolidtoken)) {
744                                if (!(TBaseType.mysametext(lcnextsolidtoken.getAstext(), "tran")
745                                        || TBaseType.mysametext(lcnextsolidtoken.getAstext(), "transaction")
746                                        || TBaseType.mysametext(lcnextsolidtoken.getAstext(), "distributed")
747                                        || TBaseType.mysametext(lcnextsolidtoken.getAstext(), "dialog")
748                                        || TBaseType.mysametext(lcnextsolidtoken.getAstext(), "conversation")
749                                ))
750                                    lcblocklevel++;
751                            } else
752                                lcblocklevel++;
753
754                        } else if (ast.tokencode == TBaseType.rrw_case)
755                            lcblocklevel++;
756                        else if (ast.tokencode == TBaseType.rrw_when) {
757                            // Each WHEN expects a matching THEN (CASE, MERGE,
758                            // EXCEPTION). Track so that THEN does not get
759                            // confused with the Watcom IF...THEN opener.
760                            lcwhenPending++;
761                        } else if (ast.tokencode == TBaseType.rrw_then) {
762                            if (lcwhenPending > 0)
763                                lcwhenPending--;
764                            else
765                                lcwatcomIfNest++;
766                        } else if (ast.tokencode == flexer.getkeywordvalue("ENDIF")) {
767                            // One-word ENDIF closes a Watcom IF if one is open.
768                            if (lcwatcomIfNest > 0)
769                                lcwatcomIfNest--;
770                        } else if (ast.tokencode == TBaseType.rrw_end) {
771
772                            lcisendconversation = false;
773                            // Watcom/SQL Anywhere/Sybase IQ: `END IF` closes a
774                            // Watcom IF statement only when there is an open
775                            // IF...THEN to match. Without that match, `END`
776                            // followed by `IF` is just a block-close followed
777                            // by a new T-SQL `IF` statement (semicolons are
778                            // optional in Sybase/T-SQL), so we must still
779                            // decrement the block level normally.
780                            boolean lcisendif = false;
781
782                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i, 1, false);
783                            if (TBaseType.assigned(lcnextsolidtoken)) {
784                                if (lcnextsolidtoken.tokencode == flexer.getkeywordvalue("CONVERSATION"))
785                                    lcisendconversation = true;
786                                else if (lcnextsolidtoken.tokencode == TBaseType.rrw_if
787                                        && lcwatcomIfNest > 0) {
788                                    lcisendif = true;
789                                    lcwatcomIfNest--;
790                                }
791                            }
792
793                            if (!lcisendconversation && !lcisendif) {
794
795                                if (lcblocklevel == 0) {
796                                    if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif) {
797                                        if (TBaseType.assigned(lcnextsolidtoken)) {
798                                            if (lcnextsolidtoken.tokencode == TBaseType.rrw_else) {
799                                                gst = EFindSqlStateType.stsql;
800                                            } else if (lcnextsolidtoken.tokentype == ETokenType.ttsemicolon) {
801                                                lcnnextsolidtoken = sourcetokenlist.nextsolidtoken(lcnextsolidtoken.posinlist, 1, false);
802                                                if (TBaseType.assigned(lcnnextsolidtoken)) {
803                                                    if (lcnnextsolidtoken.tokencode == TBaseType.rrw_else) {
804                                                        gst = EFindSqlStateType.stsql;
805                                                    }
806                                                }
807                                            }
808                                        }
809                                    }
810
811                                    if (gst != EFindSqlStateType.stsql) {
812                                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
813                                        gst = EFindSqlStateType.stnormal;
814                                    }
815
816                                } else {
817                                    lcblocklevel--;
818                                }
819
820                            }
821                        }
822                    }
823                    break;
824                }
825                case sttrycatch: {
826
827                    if (TBaseType.assigned(lcnextsqlstatement)) {
828                        if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo) {
829                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
830                            gcurrentsqlstatement = lcnextsqlstatement;
831                            addTokenToStatement(ast);
832                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
833                            gst = EFindSqlStateType.stnormal;
834                        } else {
835                            if (
836                                    (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry)
837                                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
838                            )
839                                lctrycatchlevel++;
840                            lcnextsqlstatement = null;
841                        }
842                    }
843
844                    if (gst == EFindSqlStateType.sttrycatch) {
845                        addTokenToStatement(ast);
846                        if ((ast.tokencode == TBaseType.rrw_try) ||
847                                (ast.tokencode == TBaseType.rrw_catch)) {
848                            if (TBaseType.assigned(lcprevsolidtoken)) {
849                                if (lcprevsolidtoken.tokencode == TBaseType.rrw_end) {
850                                    if (lctrycatchlevel == 0) {
851                                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
852                                        gst = EFindSqlStateType.stnormal;
853                                    } else
854                                        lctrycatchlevel--;
855                                }
856                            }
857                        }
858                    }
859                    break;
860                }
861                case stsql: {
862                    if ((ast.tokentype == ETokenType.ttsemicolon)) {
863                        lcstillinsql = false;
864                        if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif) {
865                            lcnextsolidtoken = sourcetokenlist.nextsolidtoken(i, 1, false);
866                            if (TBaseType.assigned(lcnextsolidtoken)) {
867                                if (lcnextsolidtoken.tokencode == TBaseType.rrw_else) {
868                                    addTokenToStatement(ast);
869                                    lcstillinsql = true;
870                                }
871
872                            }
873                        }
874
875                        if (!lcstillinsql) {
876                            gst = EFindSqlStateType.stnormal;
877                            addTokenToStatement(ast);
878                            gcurrentsqlstatement.semicolonended = ast;
879                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
880                        }
881
882                    } else if (TBaseType.assigned(lcnextsqlstatement)) {
883
884                        if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo) {
885                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
886                            gcurrentsqlstatement = lcnextsqlstatement;
887                            addTokenToStatement(ast);
888                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
889                            gst = EFindSqlStateType.stnormal;
890                            continue;
891                        }
892
893                        switch (gcurrentsqlstatement.sqlstatementtype) {
894                            case sstmssqlif:
895                            case sstmssqlwhile: {
896                                if ((lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegincatch)
897                                        || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlbegintry)) {
898                                    addTokenToStatement(ast);
899                                    gst = EFindSqlStateType.stblock;
900                                    lcblocklevel = 1;
901                                    lcnextsqlstatement = null;
902                                    continue;
903
904                                } else if (gcurrentsqlstatement.dummytag == 1) {
905                                    addTokenToStatement(ast);
906
907                                    if ((lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
908                                            || (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile))
909                                        gcurrentsqlstatement.dummytag = 1;
910                                    else
911                                        gcurrentsqlstatement.dummytag = 0;
912
913
914                                    lcnextsqlstatement = null;
915                                    continue;
916                                }
917                                break;
918                            }
919                            case sstmssqlalterqueue: {
920                                if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlexec) {
921                                    addTokenToStatement(ast);
922
923                                    lcnextsqlstatement = null;
924                                    continue;
925
926                                }
927                                break;
928                            }
929                            case sstmssqlcreateschema: {
930                                addTokenToStatement(ast);
931                                lcnextsqlstatement = null;
932                                continue;
933                            }
934                        }
935
936                        onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
937                        gcurrentsqlstatement = lcnextsqlstatement;
938                        addTokenToStatement(ast);
939
940                        switch (gcurrentsqlstatement.sqlstatementtype) {
941                            case sstmssqlcreateprocedure:
942                            case sstmssqlcreatefunction:
943                            case sstcreatetrigger:
944                            case sstmssqlalterprocedure:
945                            case sstmssqlalterfunction:
946                            case sstmssqlaltertrigger: {
947                                gst = EFindSqlStateType.ststoredprocedure;
948                                break;
949                            }
950                            case sstmssqlbegintry:
951                            case sstmssqlbegincatch: {
952                                gst = EFindSqlStateType.sttrycatch;
953                                lctrycatchlevel = 0;
954                                break;
955                            }
956                            case sstmssqlgo: {
957                                gst = EFindSqlStateType.stnormal;
958                                break;
959                            }
960                            default: {
961                                gst = EFindSqlStateType.stsql;
962                                break;
963                            }
964                        }
965
966                    } else if ((ast.tokencode == TBaseType.rrw_begin)) {
967                        if ((gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
968                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)) {
969                            gst = EFindSqlStateType.stblock;
970                            lcblocklevel = 0;
971                            addTokenToStatement(ast);
972                        } else if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqldeclare) {
973                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
974                            gcurrentsqlstatement = new TMssqlBlock(vendor);
975                            gcurrentsqlstatement.sqlstatementtype = ESqlStatementType.sstmssqlblock;
976                            addTokenToStatement(ast);
977                            gst = EFindSqlStateType.stblock;
978                        } else {
979                            addTokenToStatement(ast);
980                        }
981                    } else if ((ast.tokencode == TBaseType.rrw_case)) {
982                        case_end_nest++;
983                        addTokenToStatement(ast);
984                    } else if ((ast.tokencode == TBaseType.rrw_end)) {
985                        if (case_end_nest > 0) {
986                            case_end_nest--;
987                        }
988                        addTokenToStatement(ast);
989                    } else if ((ast.tokencode == TBaseType.rrw_else)) {
990                        addTokenToStatement(ast);
991                        if (((gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlif)
992                                || (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlwhile)
993                        ) && (case_end_nest == 0))
994                            gcurrentsqlstatement.dummytag = 1;
995                    } else {
996                        addTokenToStatement(ast);
997                        if (gcurrentsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlset) {
998                            if (ast.tokencode == TBaseType.rrw_on) {
999                                gst = EFindSqlStateType.stnormal;
1000                                onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
1001                            }
1002                        }
1003                    }
1004                    break;
1005                }
1006                case ststoredprocedure: {
1007                    if (TBaseType.assigned(lcnextsqlstatement)) {
1008                        if (lcnextsqlstatement.sqlstatementtype == ESqlStatementType.sstmssqlgo) {
1009                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
1010                            gcurrentsqlstatement = lcnextsqlstatement;
1011                            addTokenToStatement(ast);
1012                            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
1013                            gst = EFindSqlStateType.stnormal;
1014                        } else {
1015                            gst = EFindSqlStateType.ststoredprocedurebody;
1016                            addTokenToStatement(ast);
1017
1018                            lcnextsqlstatement = null;
1019                        }
1020                    }
1021
1022                    if (gst == EFindSqlStateType.ststoredprocedure) {
1023                        addTokenToStatement(ast);
1024                        if (ast.tokencode == TBaseType.rrw_begin) {
1025                            gst = EFindSqlStateType.stblock;
1026                        }
1027                    }
1028                    break;
1029                }
1030                case ststoredprocedurebody: {
1031                    if (TBaseType.assigned(lcnextsqlstatement)) {
1032                        switch (lcnextsqlstatement.sqlstatementtype) {
1033                            case sstmssqlgo: {
1034                                onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
1035                                gcurrentsqlstatement = lcnextsqlstatement;
1036                                addTokenToStatement(ast);
1037                                onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
1038                                gst = EFindSqlStateType.stnormal;
1039                                break;
1040                            }
1041                            case sstmssqlcreateprocedure:
1042                            case sstmssqlcreatefunction:
1043                            case sstcreatetrigger:
1044                            case sstmssqlalterprocedure:
1045                            case sstmssqlalterfunction:
1046                            case sstmssqlaltertrigger: {
1047                                onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, false, builder);
1048                                gcurrentsqlstatement = lcnextsqlstatement;
1049                                addTokenToStatement(ast);
1050                                gst = EFindSqlStateType.ststoredprocedure;
1051                                break;
1052                            }
1053                            default: {
1054                                lcnextsqlstatement = null;
1055                                break;
1056                            }
1057                        }
1058                    }
1059
1060                    if (gst == EFindSqlStateType.ststoredprocedurebody)
1061                        addTokenToStatement(ast);
1062                }
1063                break;
1064            }
1065        }
1066
1067        // last statement
1068        if (TBaseType.assigned(gcurrentsqlstatement) && (gst != EFindSqlStateType.stnormal)) {
1069            onRawStatementComplete(parserContext, gcurrentsqlstatement, fparser, null, sqlstatements, true, builder);
1070        }
1071
1072        builder.sqlStatements(this.sqlstatements);
1073        builder.errorCode(errorcount);
1074        builder.errorMessage(errorcount == 0 ? "" : String.format("Extraction completed with %d error(s)", errorcount));
1075    }
1076}