001package gudusoft.gsqlparser.dlineage.dataflow.model;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.ESetOperatorType;
005import gudusoft.gsqlparser.TCustomSqlStatement;
006import gudusoft.gsqlparser.TSourceToken;
007import gudusoft.gsqlparser.dlineage.util.DlineageUtil;
008import gudusoft.gsqlparser.dlineage.util.Pair3;
009import gudusoft.gsqlparser.nodes.TCTE;
010import gudusoft.gsqlparser.nodes.TFunctionCall;
011import gudusoft.gsqlparser.nodes.TJoinList;
012import gudusoft.gsqlparser.pp.utils.SourceTokenSearcher;
013import gudusoft.gsqlparser.sqlenv.TSQLEnv;
014import gudusoft.gsqlparser.stmt.*;
015import gudusoft.gsqlparser.stmt.hive.THiveLoad;
016import gudusoft.gsqlparser.stmt.mssql.TCreateExternalDataSourceStmt;
017import gudusoft.gsqlparser.stmt.mssql.TMssqlCreateFunction;
018import gudusoft.gsqlparser.stmt.mssql.TMssqlDeclare;
019import gudusoft.gsqlparser.stmt.mssql.TMssqlExecute;
020import gudusoft.gsqlparser.stmt.snowflake.TCreateStageStmt;
021import gudusoft.gsqlparser.stmt.snowflake.TCreateStreamStmt;
022import gudusoft.gsqlparser.stmt.snowflake.TSnowflakeCopyIntoStmt;
023import gudusoft.gsqlparser.stmt.teradata.TTeradataCreateProcedure;
024import gudusoft.gsqlparser.util.Logger;
025import gudusoft.gsqlparser.util.LoggerFactory;
026import gudusoft.gsqlparser.util.SQLUtil;
027
028import java.io.ByteArrayInputStream;
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.List;
032import java.util.Properties;
033
034public class Process {
035    private static final Logger logger = LoggerFactory.getLogger(Process.class);
036    private long id;
037    protected String server;
038    protected String schema;
039    protected String database;
040    private String name;
041    private String procedureName;
042    private Long procedureId;
043    private String queryHashId;
044    private String type;
045    private String customType;
046    private TCustomSqlStatement gspObject;
047    private Pair3<Long, Long, String> startPosition;
048    private Pair3<Long, Long, String> endPosition;
049    private List<Object> targetColumns = new ArrayList<Object>();
050    private List<Transform> transforms = new ArrayList<Transform>();
051
052    public Process(TCustomSqlStatement gspObject) {
053
054        if (gspObject == null) {
055            throw new IllegalArgumentException("Process arguments can't be null.");
056        }
057
058        id = ++ModelBindingManager.get().TABLE_COLUMN_ID;
059
060        this.gspObject = gspObject;
061
062        TSourceToken startToken = gspObject.getStartToken();
063        TSourceToken endToken = gspObject.getEndToken();
064        if (startToken != null) {
065            this.startPosition = new Pair3<Long, Long, String>(startToken.lineNo, startToken.columnNo,
066                    ModelBindingManager.getGlobalHash());
067        }
068
069        if (endToken != null) {
070            this.endPosition = new Pair3<Long, Long, String>(endToken.lineNo,
071                    endToken.columnNo + SQLUtil.endTrim(endToken.astext).length(), ModelBindingManager.getGlobalHash());
072        }
073
074        this.schema = ModelBindingManager.getGlobalSchema();
075        this.database = ModelBindingManager.getGlobalDatabase();
076        this.queryHashId = SQLUtil.stringToMD5(gspObject.asCanonical().replaceAll("\r?\n", "\r\n"));
077
078        EDbVendor vendor = ModelBindingManager.getGlobalOption().getVendor();
079        boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
080        boolean supportSchema = TSQLEnv.supportSchema(vendor);
081
082        fillSchemaInfo();
083
084        if (!supportCatalog) {
085            this.database = null;
086        } else if (this.database == null && !TSQLEnv.DEFAULT_DB_NAME.equals(getDefaultDatabase())) {
087            this.database = getDefaultDatabase();
088        }
089
090        if (!supportSchema) {
091            this.schema = null;
092        } else if (this.schema == null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equals(getDefaultSchema())) {
093            this.schema = getDefaultSchema();
094        }
095
096        String procedureParent = getProcedureParentName(gspObject);
097        if (procedureParent != null) {
098            procedureName = procedureParent;
099            Procedure procedure = ModelBindingManager.get()
100                    .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(
101                            DlineageUtil.getProcedureNameWithArgs(getParentProcedure(gspObject))));
102            if (procedure == null) {
103                procedure = ModelBindingManager.get().getProcedureByName(DlineageUtil.getIdentifierNormalTableName(
104                        DlineageUtil.getProcedureNameWithArgNum(getParentProcedure(gspObject))));
105            }
106            if (procedure != null) {
107                this.procedureId = procedure.getId();
108            }
109        } else {
110            procedureName = "batchQueries";
111        }
112
113        TCustomSqlStatement stmt = DlineageUtil.getTopStmt(ModelBindingManager.getGlobalStmtStack().peek());
114        String sqlComment = null;
115        try {
116            sqlComment = stmt.getCommentBeforeNode();
117        } catch (Exception e) {
118        }
119        if (!SQLUtil.isEmpty(sqlComment) && sqlComment.indexOf("process") != -1) {
120            Properties properties = new Properties();
121            try {
122                properties.load(
123                        new ByteArrayInputStream(sqlComment.replace("--", "").trim().replace(",", "\n").getBytes()));
124                if (properties.containsKey("process_label")) {
125                    this.customType = properties.getProperty("process_label");
126                } else if (properties.containsKey("process")) {
127                    this.customType = properties.getProperty("process");
128                }
129            } catch (IOException e) {
130                logger.error("load sql comment properties failed.", e);
131            }
132        }
133
134        if (gspObject instanceof TCreateTableSqlStatement) {
135            this.type = "Create Table";
136        } else if (gspObject instanceof TCreateViewSqlStatement) {
137            this.type = "Create View";
138        } else if (gspObject instanceof TUpdateSqlStatement) {
139            this.type = "Update";
140        } else if (gspObject instanceof TMergeSqlStatement) {
141            this.type = "Merge";
142        } else if (gspObject instanceof TInsertSqlStatement) {
143            this.type = "Insert";
144        } else if (gspObject instanceof TSelectSqlStatement
145                && ((TSelectSqlStatement) gspObject).getIntoClause() != null) {
146            this.type = "Select Into";
147        } else if (gspObject instanceof TCreateTriggerStmt) {
148            this.type = "Create Trigger";
149        } else if (gspObject instanceof TCreateFunctionStmt) {
150            this.type = "Create Function";
151        } else if (gspObject instanceof TMssqlCreateFunction) {
152            this.type = "Create Function";
153        } else if (gspObject instanceof TCreateProcedureStmt) {
154            this.type = "Create Procedure";
155        } else if (gspObject instanceof THiveLoad) {
156            this.type = "Hive Load";
157        } else if (gspObject instanceof TSnowflakeCopyIntoStmt) {
158            this.type = "Copy Into";
159        } else if (gspObject instanceof TAlterTableStatement) {
160            this.type = "Alter Table";
161        } else if (gspObject instanceof TRenameStmt) {
162            this.type = "Rename Table";
163        } else if (gspObject instanceof TMssqlDeclare) {
164            this.type = "Mssql Declare";
165        } else if (gspObject instanceof TCreateStageStmt) {
166            this.type = "Create Stage";
167        } else if (gspObject instanceof TCreateSynonymStmt) {
168            this.type = "Create Synonym";
169        } else if (gspObject instanceof TCreateExternalDataSourceStmt) {
170            this.type = "Create Datasource";
171        } else if (gspObject instanceof TCreateStreamStmt) {
172            this.type = "Create Stream";
173        } else if (gspObject instanceof TCreateDatabaseSqlStatement) {
174            this.type = "Create Database";
175        } else if (gspObject instanceof TCreateSchemaSqlStatement) {
176            this.type = "Create Schema";
177        } else if (gspObject instanceof TMssqlExecute && ((TMssqlExecute)gspObject).getModuleName().toString().equalsIgnoreCase("sp_rename")) {
178            this.type = "Rename Table";
179        } else if (gspObject instanceof TCallStatement){
180            this.type = "Function Call";
181        } else {
182            this.type = gspObject.getClass().getSimpleName();
183        }
184
185        if (this.server == null && !TSQLEnv.DEFAULT_SERVER_NAME.equals(getDefaultServer())) {
186            this.server = getDefaultServer();
187        }
188
189        if (stmt.getStatements() != null) {
190            for (int i = 0; i < stmt.getStatements().size(); i++) {
191                TCustomSqlStatement subquery = stmt.getStatements().get(i);
192                if (subquery instanceof TSelectSqlStatement) {
193                    TSelectSqlStatement select = (TSelectSqlStatement) subquery;
194                    appendTransform(select, ESetOperatorType.none);
195                }
196            }
197        }
198    }
199
200    public Process(TFunctionCall gspObject) {
201
202        if (gspObject == null) {
203            throw new IllegalArgumentException("Process arguments can't be null.");
204        }
205
206        id = ++ModelBindingManager.get().TABLE_COLUMN_ID;
207
208
209        TSourceToken startToken = gspObject.getStartToken();
210        TSourceToken endToken = gspObject.getEndToken();
211        if (startToken != null) {
212            this.startPosition = new Pair3<Long, Long, String>(startToken.lineNo, startToken.columnNo,
213                    ModelBindingManager.getGlobalHash());
214        }
215
216        if (endToken != null) {
217            this.endPosition = new Pair3<Long, Long, String>(endToken.lineNo,
218                    endToken.columnNo + SQLUtil.endTrim(endToken.astext).length(), ModelBindingManager.getGlobalHash());
219        }
220
221        TCustomSqlStatement stmt = DlineageUtil.getTopStmt(ModelBindingManager.getGlobalStmtStack().peek());
222        this.gspObject = stmt;
223
224        this.schema = ModelBindingManager.getGlobalSchema();
225        this.database = ModelBindingManager.getGlobalDatabase();
226
227        if (stmt != null) {
228            this.queryHashId = SQLUtil.stringToMD5(stmt.asCanonical().replaceAll("\r?\n", "\r\n"));
229        }
230
231        EDbVendor vendor = ModelBindingManager.getGlobalOption().getVendor();
232        boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
233        boolean supportSchema = TSQLEnv.supportSchema(vendor);
234
235        fillSchemaInfo();
236
237        if (!supportCatalog) {
238            this.database = null;
239        } else if (this.database == null && !TSQLEnv.DEFAULT_DB_NAME.equals(getDefaultDatabase())) {
240            this.database = getDefaultDatabase();
241        }
242
243        if (!supportSchema) {
244            this.schema = null;
245        } else if (this.schema == null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equals(getDefaultSchema())) {
246            this.schema = getDefaultSchema();
247        }
248
249        String procedureParent = getProcedureParentName(stmt);
250        if (procedureParent != null) {
251            procedureName = procedureParent;
252            Procedure procedure = ModelBindingManager.get()
253                    .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(
254                            DlineageUtil.getProcedureNameWithArgs(getParentProcedure(stmt))));
255            if (procedure == null) {
256                procedure = ModelBindingManager.get().getProcedureByName(DlineageUtil.getIdentifierNormalTableName(
257                        DlineageUtil.getProcedureNameWithArgNum(getParentProcedure(stmt))));
258            }
259            if (procedure != null) {
260                this.procedureId = procedure.getId();
261            }
262        } else {
263            procedureName = "batchQueries";
264        }
265
266        String sqlComment = null;
267        try {
268            sqlComment = stmt.getCommentBeforeNode();
269        } catch (Exception e) {
270        }
271        if (!SQLUtil.isEmpty(sqlComment) && sqlComment.indexOf("process") != -1) {
272            Properties properties = new Properties();
273            try {
274                properties.load(
275                        new ByteArrayInputStream(sqlComment.replace("--", "").trim().replace(",", "\n").getBytes()));
276                if (properties.containsKey("process_label")) {
277                    this.customType = properties.getProperty("process_label");
278                } else if (properties.containsKey("process")) {
279                    this.customType = properties.getProperty("process");
280                }
281            } catch (IOException e) {
282                logger.error("load sql comment properties failed.", e);
283            }
284        }
285
286        this.type = "Function Call";
287
288        if (this.server == null && !TSQLEnv.DEFAULT_SERVER_NAME.equals(getDefaultServer())) {
289            this.server = getDefaultServer();
290        }
291    }
292
293    private void appendTransform(TSelectSqlStatement select, ESetOperatorType operatorType) {
294        if (operatorType != ESetOperatorType.none) {
295            Transform transform = new Transform();
296            transform.setType(operatorType.name());
297            transform.setCodeString(select.toString());
298            transform.setStartToken(select.getStartToken());
299            transform.setEndToken(select.getEndToken());
300            transforms.add(transform);
301        } else {
302            if (select.getCteList() != null && select.getCteList().size() > 0) {
303                for (int i = 0; i < select.getCteList().size(); i++) {
304                    TCTE cte = select.getCteList().getCTE(i);
305                    Transform transform = new Transform();
306                    transform.setType(Transform.CTE);
307                    transform.setCodeString(cte.toString());
308                    transform.setStartToken(cte.getStartToken());
309                    transform.setEndToken(cte.getEndToken());
310                    transforms.add(transform);
311                }
312            }
313            if (select.getSetOperatorType() != ESetOperatorType.none) {
314                if (select.getLeftStmt() != null) {
315                    appendTransform(select.getLeftStmt(),
316                            select.getLeftStmt().getSetOperatorType() != ESetOperatorType.none ? ESetOperatorType.none
317                                    : select.getSetOperatorType());
318                }
319                if (select.getRightStmt() != null) {
320                    appendTransform(select.getRightStmt(),
321                            select.getRightStmt().getSetOperatorType() != ESetOperatorType.none ? ESetOperatorType.none
322                                    : select.getSetOperatorType());
323                }
324            } else {
325                if (select.getJoins() != null && select.getJoins().size() > 0) {
326                    TJoinList joins = select.getJoins();
327                    TSourceToken startToken = joins.getStartToken();
328                                        TSourceToken fromToken = SourceTokenSearcher.backforwardSearch(startToken, 10, "from");
329                                        if (fromToken == null) {
330                                                if (startToken.astext.equalsIgnoreCase("from")) {
331                                                        fromToken = startToken;
332                                                } else {
333                                                        return;
334                                                }
335                                        }
336                    StringBuilder builder = new StringBuilder();
337                                        if (joins.getEndToken().posinlist < fromToken.container.size()
338                                                        && joins.getEndToken().posinlist >= fromToken.posinlist) {
339                                                for (int i = fromToken.posinlist; i <= joins.getEndToken().posinlist; i++) {
340                                                        builder.append(fromToken.container.get(i));
341                                                }
342                                        }
343                                        else {
344                                                System.err.println("Handle statement transform error, statemet is:");
345                                                System.err.println(select.toString());
346                                        }
347                    Transform transform = new Transform();
348                    transform.setType(Transform.FROM);
349                    transform.setCodeString(builder.toString());
350                    transform.setStartToken(fromToken);
351                    transform.setEndToken(joins.getEndToken());
352                    transforms.add(transform);
353                }
354            }
355        }
356    }
357
358    private TSelectSqlStatement getFirstSubquery(TSelectSqlStatement select) {
359        if (select.getSetOperatorType() != ESetOperatorType.none) {
360            return getFirstSubquery(select.getLeftStmt());
361        } else {
362            return select;
363        }
364    }
365
366    private TSelectSqlStatement getLastSubquery(TSelectSqlStatement select) {
367        if (select.getSetOperatorType() != ESetOperatorType.none) {
368            return getLastSubquery(select.getRightStmt());
369        } else {
370            return select;
371        }
372    }
373
374    private void fillSchemaInfo() {
375        TCustomSqlStatement stmt = DlineageUtil.getTopStmt(ModelBindingManager.getGlobalStmtStack().peek());
376        String sqlComment = null;
377        try {
378            sqlComment = stmt.getCommentBeforeNode();
379        } catch (Exception e) {
380        }
381        if (!SQLUtil.isEmpty(sqlComment) && (sqlComment.indexOf("db") != -1 || sqlComment.indexOf("schema") != -1)) {
382            Properties properties = new Properties();
383            try {
384                properties.load(
385                        new ByteArrayInputStream(sqlComment.replace("--", "").trim().replace(",", "\n").getBytes()));
386                if (SQLUtil.isEmpty(this.server) && properties.containsKey("db-instance")) {
387                    this.server = properties.getProperty("db-instance");
388                }
389                if (SQLUtil.isEmpty(this.database) && properties.containsKey("db")) {
390                    this.database = properties.getProperty("db");
391                    if (this.database.indexOf(".") != -1) {
392                        String delimitedChar = TSQLEnv.delimitedChar(ModelBindingManager.getGlobalOption().getVendor());
393                        this.database = delimitedChar + SQLUtil.trimColumnStringQuote(this.database) + delimitedChar;
394                    }
395                }
396                if (SQLUtil.isEmpty(this.schema) && properties.containsKey("schema")) {
397                    this.schema = properties.getProperty("schema");
398                    if (this.schema.indexOf(".") != -1) {
399                        String delimitedChar = TSQLEnv.delimitedChar(ModelBindingManager.getGlobalOption().getVendor());
400                        this.schema = delimitedChar + SQLUtil.trimColumnStringQuote(this.schema) + delimitedChar;
401                    }
402                }
403            } catch (IOException e) {
404                logger.error("load sql comment properties failed.", e);
405            }
406        }
407    }
408
409    protected String getDefaultServer() {
410        String defaultServer = null;
411        if (ModelBindingManager.getGlobalSQLEnv() != null) {
412            defaultServer = ModelBindingManager.getGlobalSQLEnv().getDefaultServerName();
413        }
414        if (!SQLUtil.isEmpty(defaultServer))
415            return defaultServer;
416        return TSQLEnv.DEFAULT_SERVER_NAME;
417    }
418
419    protected String getDefaultSchema() {
420        String defaultSchema = null;
421        if (ModelBindingManager.getGlobalSQLEnv() != null) {
422            defaultSchema = ModelBindingManager.getGlobalSQLEnv().getDefaultSchemaName();
423        }
424        if (!SQLUtil.isEmpty(defaultSchema))
425            return defaultSchema;
426        return TSQLEnv.DEFAULT_SCHEMA_NAME;
427    }
428
429    protected String getDefaultDatabase() {
430        String defaultDatabase = null;
431        if (ModelBindingManager.getGlobalSQLEnv() != null) {
432            defaultDatabase = ModelBindingManager.getGlobalSQLEnv().getDefaultCatalogName();
433        }
434        if (!SQLUtil.isEmpty(defaultDatabase))
435            return defaultDatabase;
436        return TSQLEnv.DEFAULT_DB_NAME;
437    }
438
439    private String getProcedureParentName(TCustomSqlStatement stmt) {
440        if (stmt instanceof TStoredProcedureSqlStatement) {
441                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
442                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
443                        }
444                }
445
446        if (stmt == null) {
447                return null;
448        }
449        
450        if (stmt instanceof TTeradataCreateProcedure) {
451            if (((TTeradataCreateProcedure) stmt).getProcedureName() != null) {
452                return ((TTeradataCreateProcedure) stmt).getProcedureName().toString();
453            }
454        }
455        
456        if (stmt instanceof TStoredProcedureSqlStatement) {
457            if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
458                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
459            }
460
461            if (stmt instanceof TCommonBlock) {
462                if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
463                    stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
464                }
465                else {
466                        return getProcedureParentName(stmt.getParentStmt());
467                }
468            }
469            
470            return null;
471        }
472        
473                stmt = stmt.getParentStmt();
474        if (stmt == null) {
475            if(ModelBindingManager.getGlobalProcedure()!=null) {
476                return getProcedureParentName(ModelBindingManager.getGlobalProcedure());
477            }
478            else {
479                return null;
480            }
481        }
482        
483
484        if (stmt instanceof TCommonBlock) {
485            if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
486                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
487            }
488        }
489
490        return getProcedureParentName(stmt);
491    }
492
493    private TStoredProcedureSqlStatement getParentProcedure(TCustomSqlStatement stmt) {
494        if (stmt == null) {
495                return null;
496        }
497        
498        if (stmt instanceof TStoredProcedureSqlStatement) {
499            if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
500                return ((TStoredProcedureSqlStatement) stmt);
501            }
502            
503            if (stmt instanceof TCommonBlock) {
504                if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
505                    return (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
506                }
507                else {
508                        return getParentProcedure(stmt.getParentStmt());
509                }
510            }
511            
512            return null;
513        }
514        
515            stmt = stmt.getParentStmt();
516        if (stmt == null) {
517            if(ModelBindingManager.getGlobalProcedure()!=null) {
518                return ModelBindingManager.getGlobalProcedure();
519            }
520            else {
521                return null;
522            }
523        }
524        
525        if (stmt instanceof TCommonBlock) {
526            if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
527                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
528            }
529        }
530
531        return getParentProcedure(stmt);
532    }
533
534    public Pair3<Long, Long, String> getStartPosition() {
535        return startPosition;
536    }
537
538    public Pair3<Long, Long, String> getEndPosition() {
539        return endPosition;
540    }
541
542    public TCustomSqlStatement getGspObject() {
543        return gspObject;
544    }
545
546    public long getId() {
547        return id;
548    }
549
550    public String getSchema() {
551        return schema;
552    }
553
554    public String getDatabase() {
555        return database;
556    }
557
558    public String getName() {
559        return name;
560    }
561
562    public String getType() {
563        return type;
564    }
565
566    public void setName(String name) {
567        this.name = name;
568    }
569
570    public void setType(String type) {
571        this.type = type;
572    }
573
574    public String getProcedureName() {
575        return procedureName;
576    }
577
578    public void setProcedureName(String procedureName) {
579        this.procedureName = procedureName;
580    }
581
582    public String getQueryHashId() {
583        return queryHashId;
584    }
585
586    public void setQueryHashId(String queryHashId) {
587        this.queryHashId = queryHashId;
588    }
589
590    public void appendColumn(Object tableColumn) {
591        if (tableColumn != null && !targetColumns.contains(tableColumn)) {
592            targetColumns.add(tableColumn);
593        }
594    }
595
596    public List<Object> getTargetColumns() {
597        return targetColumns;
598    }
599
600    public String getCustomType() {
601        return customType;
602    }
603
604    public Long getProcedureId() {
605        return procedureId;
606    }
607
608    public String getServer() {
609        return server;
610    }
611
612    public List<Transform> getTransforms() {
613        return transforms;
614    }
615}