Source code

001package gudusoft.gsqlparser.dlineage.dataflow.model;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.ESetOperatorType;
005import gudusoft.gsqlparser.TCustomSqlStatement;
006import gudusoft.gsqlparser.TSourceToken;
007import gudusoft.gsqlparser.dlineage.util.DlineageUtil;
008import gudusoft.gsqlparser.dlineage.util.Pair3;
009import gudusoft.gsqlparser.nodes.TCTE;
010import gudusoft.gsqlparser.nodes.TFunctionCall;
011import gudusoft.gsqlparser.nodes.TJoinList;
012import gudusoft.gsqlparser.pp.utils.SourceTokenSearcher;
013import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType;
014import gudusoft.gsqlparser.sqlenv.TSQLEnv;
015import gudusoft.gsqlparser.stmt.*;
016import gudusoft.gsqlparser.stmt.hive.THiveLoad;
017import gudusoft.gsqlparser.stmt.mssql.TCreateExternalDataSourceStmt;
018import gudusoft.gsqlparser.stmt.mssql.TMssqlCreateFunction;
019import gudusoft.gsqlparser.stmt.mssql.TMssqlDeclare;
020import gudusoft.gsqlparser.stmt.mssql.TMssqlExecute;
021import gudusoft.gsqlparser.stmt.snowflake.TCreateStageStmt;
022import gudusoft.gsqlparser.stmt.snowflake.TCreateStreamStmt;
023import gudusoft.gsqlparser.stmt.snowflake.TSnowflakeCopyIntoStmt;
024import gudusoft.gsqlparser.stmt.teradata.TTeradataCreateProcedure;
025import gudusoft.gsqlparser.util.Logger;
026import gudusoft.gsqlparser.util.LoggerFactory;
027import gudusoft.gsqlparser.util.SQLUtil;
028
029import java.io.ByteArrayInputStream;
030import java.io.IOException;
031import java.util.ArrayDeque;
032import java.util.ArrayList;
033import java.util.Deque;
034import java.util.List;
035import java.util.Properties;
036
037public class Process {
038    private static final Logger logger = LoggerFactory.getLogger(Process.class);
039    private long id;
040    protected String server;
041    protected String schema;
042    protected String database;
043    private String name;
044    private String procedureName;
045    private Long procedureId;
046    private String queryHashId;
047    private String type;
048    private String customType;
049    private TCustomSqlStatement gspObject;
050    private Pair3<Long, Long, String> startPosition;
051    private Pair3<Long, Long, String> endPosition;
052    private List<Object> targetColumns = new ArrayList<Object>();
053    private List<Transform> transforms = new ArrayList<Transform>();
054
055    public Process(TCustomSqlStatement gspObject) {
056
057        if (gspObject == null) {
058            throw new IllegalArgumentException("Process arguments can't be null.");
059        }
060
061        id = ++ModelBindingManager.get().TABLE_COLUMN_ID;
062
063        this.gspObject = gspObject;
064
065        TSourceToken startToken = gspObject.getStartToken();
066        TSourceToken endToken = gspObject.getEndToken();
067        if (startToken != null) {
068            this.startPosition = new Pair3<Long, Long, String>(startToken.lineNo, startToken.columnNo,
069                    ModelBindingManager.getGlobalHash());
070        }
071
072        if (endToken != null) {
073            this.endPosition = new Pair3<Long, Long, String>(endToken.lineNo,
074                    endToken.columnNo + SQLUtil.endTrim(endToken.getAstext()).length(), ModelBindingManager.getGlobalHash());
075        }
076
077        this.schema = ModelBindingManager.getGlobalSchema();
078        this.database = ModelBindingManager.getGlobalDatabase();
079        this.queryHashId = SQLUtil.stringToMD5(gspObject.asCanonical().replaceAll("\r?\n", "\r\n"));
080
081        EDbVendor vendor = ModelBindingManager.getGlobalOption().getVendor();
082        boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
083        boolean supportSchema = TSQLEnv.supportSchema(vendor);
084
085        fillSchemaInfo();
086
087        if (!supportCatalog) {
088            this.database = null;
089        } else if (this.database == null && !TSQLEnv.DEFAULT_DB_NAME.equals(getDefaultDatabase())) {
090            this.database = getDefaultDatabase();
091        }
092
093        if (!supportSchema) {
094            this.schema = null;
095        } else if (this.schema == null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equals(getDefaultSchema())) {
096            this.schema = getDefaultSchema();
097        }
098
099        String procedureParent = getProcedureParentName(gspObject);
100        if (procedureParent != null) {
101            procedureName = procedureParent;
102            Procedure procedure = ModelBindingManager.get().getRoutineModel(getParentProcedure(gspObject));
103            if (procedure != null) {
104                this.procedureId = procedure.getId();
105            }
106        } else {
107            procedureName = "batchQueries";
108        }
109
110        TCustomSqlStatement stmt = DlineageUtil.getTopStmt(ModelBindingManager.getGlobalStmtStack().peek());
111        String sqlComment = null;
112        try {
113            sqlComment = stmt.getCommentBeforeNode();
114        } catch (Exception e) {
115        }
116        if (!SQLUtil.isEmpty(sqlComment) && sqlComment.indexOf("process") != -1) {
117            Properties properties = new Properties();
118            try {
119                properties.load(
120                        new ByteArrayInputStream(sqlComment.replace("--", "").trim().replace(",", "\n").getBytes()));
121                if (properties.containsKey("process_label")) {
122                    this.customType = properties.getProperty("process_label");
123                } else if (properties.containsKey("process")) {
124                    this.customType = properties.getProperty("process");
125                }
126            } catch (IOException e) {
127                logger.error("load sql comment properties failed.", e);
128            }
129        }
130
131        if (gspObject instanceof TCreateTableSqlStatement) {
132            this.type = "Create Table";
133        } else if (gspObject instanceof TCreateViewSqlStatement) {
134            this.type = "Create View";
135        } else if (gspObject instanceof TUpdateSqlStatement) {
136            this.type = "Update";
137        } else if (gspObject instanceof TMergeSqlStatement) {
138            this.type = "Merge";
139        } else if (gspObject instanceof TInsertSqlStatement) {
140            this.type = "Insert";
141        } else if (gspObject instanceof TSelectSqlStatement
142                && ((TSelectSqlStatement) gspObject).getIntoClause() != null) {
143            this.type = "Select Into";
144        } else if (gspObject instanceof TCreateTriggerStmt) {
145            this.type = "Create Trigger";
146        } else if (gspObject instanceof TCreateFunctionStmt) {
147            this.type = "Create Function";
148        } else if (gspObject instanceof TMssqlCreateFunction) {
149            this.type = "Create Function";
150        } else if (gspObject instanceof TCreateProcedureStmt) {
151            this.type = "Create Procedure";
152        } else if (gspObject instanceof THiveLoad) {
153            this.type = "Hive Load";
154        } else if (gspObject instanceof TSnowflakeCopyIntoStmt) {
155            this.type = "Copy Into";
156        } else if (gspObject instanceof TAlterTableStatement) {
157            this.type = "Alter Table";
158        } else if (gspObject instanceof TRenameStmt) {
159            this.type = "Rename Table";
160        } else if (gspObject instanceof TMssqlDeclare) {
161            this.type = "Mssql Declare";
162        } else if (gspObject instanceof TCreateStageStmt) {
163            this.type = "Create Stage";
164        } else if (gspObject instanceof TCreateSynonymStmt) {
165            this.type = "Create Synonym";
166        } else if (gspObject instanceof TCreateExternalDataSourceStmt) {
167            this.type = "Create Datasource";
168        } else if (gspObject instanceof TCreateStreamStmt) {
169            this.type = "Create Stream";
170        } else if (gspObject instanceof TCreateDatabaseSqlStatement) {
171            this.type = "Create Database";
172        } else if (gspObject instanceof TCreateSchemaSqlStatement) {
173            this.type = "Create Schema";
174        } else if (gspObject instanceof TMssqlExecute && ((TMssqlExecute)gspObject).getModuleName().toString().equalsIgnoreCase("sp_rename")) {
175            this.type = "Rename Table";
176        } else if (gspObject instanceof TCallStatement){
177            this.type = "Function Call";
178        } else {
179            this.type = gspObject.getClass().getSimpleName();
180        }
181
182        if (this.server == null && !TSQLEnv.DEFAULT_SERVER_NAME.equals(getDefaultServer())) {
183            this.server = getDefaultServer();
184        }
185
186        if (stmt.getStatements() != null) {
187            for (int i = 0; i < stmt.getStatements().size(); i++) {
188                TCustomSqlStatement subquery = stmt.getStatements().get(i);
189                if (subquery instanceof TSelectSqlStatement) {
190                    TSelectSqlStatement select = (TSelectSqlStatement) subquery;
191                    appendTransform(select, ESetOperatorType.none);
192                }
193            }
194        }
195    }
196
197    public Process(TFunctionCall gspObject) {
198
199        if (gspObject == null) {
200            throw new IllegalArgumentException("Process arguments can't be null.");
201        }
202
203        id = ++ModelBindingManager.get().TABLE_COLUMN_ID;
204
205
206        TSourceToken startToken = gspObject.getStartToken();
207        TSourceToken endToken = gspObject.getEndToken();
208        if (startToken != null) {
209            this.startPosition = new Pair3<Long, Long, String>(startToken.lineNo, startToken.columnNo,
210                    ModelBindingManager.getGlobalHash());
211        }
212
213        if (endToken != null) {
214            this.endPosition = new Pair3<Long, Long, String>(endToken.lineNo,
215                    endToken.columnNo + SQLUtil.endTrim(endToken.getAstext()).length(), ModelBindingManager.getGlobalHash());
216        }
217
218        TCustomSqlStatement stmt = DlineageUtil.getTopStmt(ModelBindingManager.getGlobalStmtStack().peek());
219        this.gspObject = stmt;
220
221        this.schema = ModelBindingManager.getGlobalSchema();
222        this.database = ModelBindingManager.getGlobalDatabase();
223
224        if (stmt != null) {
225            this.queryHashId = SQLUtil.stringToMD5(stmt.asCanonical().replaceAll("\r?\n", "\r\n"));
226        }
227
228        EDbVendor vendor = ModelBindingManager.getGlobalOption().getVendor();
229        boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
230        boolean supportSchema = TSQLEnv.supportSchema(vendor);
231
232        fillSchemaInfo();
233
234        if (!supportCatalog) {
235            this.database = null;
236        } else if (this.database == null && !TSQLEnv.DEFAULT_DB_NAME.equals(getDefaultDatabase())) {
237            this.database = getDefaultDatabase();
238        }
239
240        if (!supportSchema) {
241            this.schema = null;
242        } else if (this.schema == null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equals(getDefaultSchema())) {
243            this.schema = getDefaultSchema();
244        }
245
246        String procedureParent = getProcedureParentName(stmt);
247        if (procedureParent != null) {
248            procedureName = procedureParent;
249            Procedure procedure = ModelBindingManager.get().getRoutineModel(getParentProcedure(stmt));
250            if (procedure != null) {
251                this.procedureId = procedure.getId();
252            }
253        } else {
254            procedureName = "batchQueries";
255        }
256
257        String sqlComment = null;
258        try {
259            sqlComment = stmt.getCommentBeforeNode();
260        } catch (Exception e) {
261        }
262        if (!SQLUtil.isEmpty(sqlComment) && sqlComment.indexOf("process") != -1) {
263            Properties properties = new Properties();
264            try {
265                properties.load(
266                        new ByteArrayInputStream(sqlComment.replace("--", "").trim().replace(",", "\n").getBytes()));
267                if (properties.containsKey("process_label")) {
268                    this.customType = properties.getProperty("process_label");
269                } else if (properties.containsKey("process")) {
270                    this.customType = properties.getProperty("process");
271                }
272            } catch (IOException e) {
273                logger.error("load sql comment properties failed.", e);
274            }
275        }
276
277        this.type = "Function Call";
278
279        if (this.server == null && !TSQLEnv.DEFAULT_SERVER_NAME.equals(getDefaultServer())) {
280            this.server = getDefaultServer();
281        }
282    }
283
284    private void appendTransform(TSelectSqlStatement select, ESetOperatorType operatorType) {
285        // Iterative traversal of the UNION tree to avoid StackOverflow with deeply nested unions.
286        Deque<Object[]> stack = new ArrayDeque<>();
287        stack.push(new Object[]{select, operatorType});
288
289        while (!stack.isEmpty()) {
290            Object[] entry = stack.pop();
291            TSelectSqlStatement current = (TSelectSqlStatement) entry[0];
292            ESetOperatorType currentOp = (ESetOperatorType) entry[1];
293
294            if (currentOp != ESetOperatorType.none) {
295                Transform transform = new Transform();
296                transform.setType(currentOp.name());
297                transform.setCodeString(current.toString());
298                transform.setStartToken(current.getStartToken());
299                transform.setEndToken(current.getEndToken());
300                transforms.add(transform);
301            } else {
302                if (current.getCteList() != null && current.getCteList().size() > 0) {
303                    for (int i = 0; i < current.getCteList().size(); i++) {
304                        TCTE cte = current.getCteList().getCTE(i);
305                        Transform transform = new Transform();
306                        transform.setType(Transform.CTE);
307                        transform.setCodeString(cte.toString());
308                        transform.setStartToken(cte.getStartToken());
309                        transform.setEndToken(cte.getEndToken());
310                        transforms.add(transform);
311                    }
312                }
313                if (current.getSetOperatorType() != ESetOperatorType.none) {
314                    // Push right first so left is processed first (stack is LIFO)
315                    if (current.getRightStmt() != null) {
316                        ESetOperatorType rightOp = current.getRightStmt().getSetOperatorType() != ESetOperatorType.none
317                                ? ESetOperatorType.none : current.getSetOperatorType();
318                        stack.push(new Object[]{current.getRightStmt(), rightOp});
319                    }
320                    if (current.getLeftStmt() != null) {
321                        ESetOperatorType leftOp = current.getLeftStmt().getSetOperatorType() != ESetOperatorType.none
322                                ? ESetOperatorType.none : current.getSetOperatorType();
323                        stack.push(new Object[]{current.getLeftStmt(), leftOp});
324                    }
325                } else {
326                    if (current.getJoins() != null && current.getJoins().size() > 0) {
327                        TJoinList joins = current.getJoins();
328                        TSourceToken startToken = joins.getStartToken();
329                        TSourceToken fromToken = SourceTokenSearcher.backforwardSearch(startToken, 10, "from");
330                        if (fromToken == null) {
331                            if (startToken.getAstext().equalsIgnoreCase("from")) {
332                                fromToken = startToken;
333                            } else {
334                                continue;
335                            }
336                        }
337                        StringBuilder builder = new StringBuilder();
338                        if (joins.getEndToken().posinlist < fromToken.container.size()
339                                && joins.getEndToken().posinlist >= fromToken.posinlist) {
340                            for (int j = fromToken.posinlist; j <= joins.getEndToken().posinlist; j++) {
341                                builder.append(fromToken.container.get(j));
342                            }
343                        } else {
344                            System.err.println("Handle statement transform error, statemet is:");
345                            System.err.println(current.toString());
346                        }
347                        Transform transform = new Transform();
348                        transform.setType(Transform.FROM);
349                        transform.setCodeString(builder.toString());
350                        transform.setStartToken(fromToken);
351                        transform.setEndToken(joins.getEndToken());
352                        transforms.add(transform);
353                    }
354                }
355            }
356        }
357    }
358
359    private TSelectSqlStatement getFirstSubquery(TSelectSqlStatement select) {
360        // Iterative left-chain walk to avoid StackOverflow with deeply nested unions.
361        TSelectSqlStatement current = select;
362        while (current.getSetOperatorType() != ESetOperatorType.none) {
363            current = current.getLeftStmt();
364        }
365        return current;
366    }
367
368    private TSelectSqlStatement getLastSubquery(TSelectSqlStatement select) {
369        // Iterative right-chain walk to avoid StackOverflow with deeply nested unions.
370        TSelectSqlStatement current = select;
371        while (current.getSetOperatorType() != ESetOperatorType.none) {
372            current = current.getRightStmt();
373        }
374        return current;
375    }
376
377    private void fillSchemaInfo() {
378        TCustomSqlStatement stmt = DlineageUtil.getTopStmt(ModelBindingManager.getGlobalStmtStack().peek());
379        String sqlComment = null;
380        try {
381            sqlComment = stmt.getCommentBeforeNode();
382        } catch (Exception e) {
383        }
384        if (!SQLUtil.isEmpty(sqlComment) && (sqlComment.indexOf("db") != -1 || sqlComment.indexOf("schema") != -1)) {
385            Properties properties = new Properties();
386            try {
387                properties.load(
388                        new ByteArrayInputStream(sqlComment.replace("--", "").trim().replace(",", "\n").getBytes()));
389                if (SQLUtil.isEmpty(this.server) && properties.containsKey("db-instance")) {
390                    this.server = properties.getProperty("db-instance");
391                }
392                if (SQLUtil.isEmpty(this.database) && properties.containsKey("db")) {
393                    this.database = properties.getProperty("db");
394                    if (this.database.indexOf(".") != -1) {
395                        this.database = SQLUtil.quoteDottedName(ModelBindingManager.getGlobalOption().getVendor(), ESQLDataObjectType.dotCatalog, this.database);
396                    }
397                }
398                if (SQLUtil.isEmpty(this.schema) && properties.containsKey("schema")) {
399                    this.schema = properties.getProperty("schema");
400                    if (this.schema.indexOf(".") != -1) {
401                        this.schema = SQLUtil.quoteDottedName(ModelBindingManager.getGlobalOption().getVendor(), ESQLDataObjectType.dotSchema, this.schema);
402                    }
403                }
404            } catch (IOException e) {
405                logger.error("load sql comment properties failed.", e);
406            }
407        }
408    }
409
410    protected String getDefaultServer() {
411        String defaultServer = null;
412        if (ModelBindingManager.getGlobalSQLEnv() != null) {
413            defaultServer = ModelBindingManager.getGlobalSQLEnv().getDefaultServerName();
414        }
415        if (!SQLUtil.isEmpty(defaultServer))
416            return defaultServer;
417        return TSQLEnv.DEFAULT_SERVER_NAME;
418    }
419
420    protected String getDefaultSchema() {
421        return ModelBindingManager.getEffectiveDefaultSchema();
422    }
423
424    protected String getDefaultDatabase() {
425        return ModelBindingManager.getEffectiveDefaultDatabase();
426    }
427
428    private String getProcedureParentName(TCustomSqlStatement stmt) {
429        return getProcedureParentName(stmt, false);
430    }
431
432    /**
433     * The name of the procedure enclosing {@code stmt}, falling back to the global procedure when the chain of parents
434     * ends without one. {@code inGlobalProcedure} is set once that fallback is taken: the global procedure's own chain
435     * can end at a statement with no parent too, and falling back again from there led straight back to the global
436     * procedure, until the stack overflowed. A Snowflake {@code CREATE TASK ... AS BEGIN ... END} does that: its block
437     * is the global procedure, has no name, and its parent, the task, is no procedure. The function is deterministic, so
438     * a second fallback never ended; every walk that ended before returns what it returned.
439     */
440    private String getProcedureParentName(TCustomSqlStatement stmt, boolean inGlobalProcedure) {
441        if (stmt instanceof TStoredProcedureSqlStatement) {
442                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
443                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
444                        }
445                }
446
447        if (stmt == null) {
448                return null;
449        }
450        
451        if (stmt instanceof TTeradataCreateProcedure) {
452            if (((TTeradataCreateProcedure) stmt).getProcedureName() != null) {
453                return ((TTeradataCreateProcedure) stmt).getProcedureName().toString();
454            }
455        }
456        
457        if (stmt instanceof TStoredProcedureSqlStatement) {
458            if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
459                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
460            }
461
462            if (stmt instanceof TCommonBlock) {
463                if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
464                    stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
465                }
466                else {
467                    return getProcedureParentName(stmt.getParentStmt(), inGlobalProcedure);
468                }
469            }
470            
471            return null;
472        }
473        
474                stmt = stmt.getParentStmt();
475        if (stmt == null) {
476            if (!inGlobalProcedure && ModelBindingManager.getGlobalProcedure() != null) {
477                return getProcedureParentName(ModelBindingManager.getGlobalProcedure(), true);
478            }
479            else {
480                return null;
481            }
482        }
483        
484
485        if (stmt instanceof TCommonBlock) {
486            if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
487                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
488            }
489        }
490
491        return getProcedureParentName(stmt, inGlobalProcedure);
492    }
493
494    private TStoredProcedureSqlStatement getParentProcedure(TCustomSqlStatement stmt) {
495        if (stmt == null) {
496                return null;
497        }
498        
499        if (stmt instanceof TStoredProcedureSqlStatement) {
500            if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
501                return ((TStoredProcedureSqlStatement) stmt);
502            }
503            
504            if (stmt instanceof TCommonBlock) {
505                if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
506                    return (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
507                }
508                else {
509                        return getParentProcedure(stmt.getParentStmt());
510                }
511            }
512            
513            return null;
514        }
515        
516            stmt = stmt.getParentStmt();
517        if (stmt == null) {
518            if(ModelBindingManager.getGlobalProcedure()!=null) {
519                return ModelBindingManager.getGlobalProcedure();
520            }
521            else {
522                return null;
523            }
524        }
525        
526        if (stmt instanceof TCommonBlock) {
527            if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
528                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
529            }
530        }
531
532        return getParentProcedure(stmt);
533    }
534
535    public Pair3<Long, Long, String> getStartPosition() {
536        return startPosition;
537    }
538
539    public Pair3<Long, Long, String> getEndPosition() {
540        return endPosition;
541    }
542
543    public TCustomSqlStatement getGspObject() {
544        return gspObject;
545    }
546
547    public long getId() {
548        return id;
549    }
550
551    public String getSchema() {
552        return schema;
553    }
554
555    public String getDatabase() {
556        return database;
557    }
558
559    public String getName() {
560        return name;
561    }
562
563    public String getType() {
564        return type;
565    }
566
567    public void setName(String name) {
568        this.name = name;
569    }
570
571    public void setType(String type) {
572        this.type = type;
573    }
574
575    public String getProcedureName() {
576        return procedureName;
577    }
578
579    public void setProcedureName(String procedureName) {
580        this.procedureName = procedureName;
581    }
582
583    public String getQueryHashId() {
584        return queryHashId;
585    }
586
587    public void setQueryHashId(String queryHashId) {
588        this.queryHashId = queryHashId;
589    }
590
591    public void appendColumn(Object tableColumn) {
592        if (tableColumn != null && !targetColumns.contains(tableColumn)) {
593            targetColumns.add(tableColumn);
594        }
595    }
596
597    public List<Object> getTargetColumns() {
598        return targetColumns;
599    }
600
601    public String getCustomType() {
602        return customType;
603    }
604
605    public Long getProcedureId() {
606        return procedureId;
607    }
608
609    public String getServer() {
610        return server;
611    }
612
613    public List<Transform> getTransforms() {
614        return transforms;
615    }
616}