Source code

001
002package gudusoft.gsqlparser.dlineage;
003
004import gudusoft.gsqlparser.*;
005import gudusoft.gsqlparser.dlineage.dataflow.listener.DataFlowHandleListener;
006import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
007import gudusoft.gsqlparser.dlineage.dataflow.metadata.grabit.GrabitMetadataAnalyzer;
008import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
009import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.SqlflowMetadataAnalyzer;
010import gudusoft.gsqlparser.dlineage.dataflow.model.*;
011import gudusoft.gsqlparser.dlineage.dataflow.model.Process;
012import gudusoft.gsqlparser.dlineage.dataflow.model.JoinRelationship.JoinClauseType;
013import gudusoft.gsqlparser.dlineage.dataflow.model.json.Coordinate;
014import gudusoft.gsqlparser.dlineage.dataflow.model.json.Dataflow;
015import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
016import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
017import gudusoft.gsqlparser.dlineage.dataflow.sqlenv.SQLEnvParser;
018import gudusoft.gsqlparser.dlineage.metadata.MetadataUtil;
019import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
020import gudusoft.gsqlparser.dlineage.util.*;
021import gudusoft.gsqlparser.nodes.TTable;
022import gudusoft.gsqlparser.nodes.*;
023import gudusoft.gsqlparser.nodes.couchbase.TObjectConstruct;
024import gudusoft.gsqlparser.nodes.couchbase.TPair;
025import gudusoft.gsqlparser.nodes.functions.TJsonObjectFunction;
026import gudusoft.gsqlparser.nodes.hive.THiveTransformClause;
027import gudusoft.gsqlparser.nodes.hive.THiveTransformClause.ETransformType;
028import gudusoft.gsqlparser.resolver2.ScopeBuildResult;
029import gudusoft.gsqlparser.resolver2.TSQLResolver2;
030import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType;
031import gudusoft.gsqlparser.sqlenv.TSQLEnv;
032import gudusoft.gsqlparser.sqlenv.TSQLSchema;
033import gudusoft.gsqlparser.sqlenv.TSQLTable;
034import gudusoft.gsqlparser.sqlenv.parser.TJSONSQLEnvParser;
035import gudusoft.gsqlparser.stmt.*;
036import gudusoft.gsqlparser.stmt.db2.TDb2ReturnStmt;
037import gudusoft.gsqlparser.stmt.db2.TDb2SqlVariableDeclaration;
038import gudusoft.gsqlparser.stmt.hive.THiveLoad;
039import gudusoft.gsqlparser.stmt.mssql.*;
040import gudusoft.gsqlparser.stmt.mysql.TLoadDataStmt;
041import gudusoft.gsqlparser.stmt.oracle.*;
042import gudusoft.gsqlparser.stmt.redshift.TRedshiftCopy;
043import gudusoft.gsqlparser.stmt.redshift.TRedshiftDeclare;
044import gudusoft.gsqlparser.stmt.snowflake.*;
045import gudusoft.gsqlparser.stmt.teradata.TTeradataCreateProcedure;
046import gudusoft.gsqlparser.util.*;
047import gudusoft.gsqlparser.util.json.JSON;
048
049import javax.script.ScriptEngine;
050import javax.script.ScriptEngineManager;
051import javax.script.ScriptException;
052import java.io.*;
053import java.util.*;
054import java.util.concurrent.atomic.AtomicInteger;
055import java.util.regex.Matcher;
056import java.util.regex.Pattern;
057import java.util.stream.Collectors;
058
059import static gudusoft.gsqlparser.EJoinType.right;
060
061@SuppressWarnings("rawtypes")
062public class DataFlowAnalyzer implements IDataFlowAnalyzer {
063
064        private static final Logger logger = LoggerFactory.getLogger(DataFlowAnalyzer.class);
065
066        private static final List<String> TERADATA_BUILTIN_FUNCTIONS = Arrays
067                        .asList(new String[] { "ACCOUNT", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
068                                        "CURRENT_USER", "DATABASE", "DATE", "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", });
069
070        private static final List<String> CONSTANT_BUILTIN_FUNCTIONS = Arrays.asList(new String[] { "ACCOUNT",
071                        "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DATABASE", "DATE",
072                        "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", "GETDATE" });
073
074        private Stack<TCustomSqlStatement> stmtStack = new Stack<TCustomSqlStatement>();
075        private List<ResultSet> appendResultSets = new ArrayList<ResultSet>();
076        private Set<TCustomSqlStatement> accessedStatements = new HashSet<TCustomSqlStatement>();
077        private Set<TSelectSqlStatement> accessedSubqueries = new HashSet<TSelectSqlStatement>();
078        private Map<String, TCustomSqlStatement> viewDDLMap = new HashMap<String, TCustomSqlStatement>();
079        private Map<String, TCustomSqlStatement> procedureDDLMap = new HashMap<String, TCustomSqlStatement>();
080        private Map<TTable, TObjectNameList> structObjectMap = new HashMap<TTable, TObjectNameList>();
081
082        private SqlInfo[] sqlInfos;
083        private List<ErrorInfo> errorInfos = new ArrayList<ErrorInfo>();
084        private IndexedLinkedHashMap<String, List<SqlInfo>> sqlInfoMap = new IndexedLinkedHashMap<String, List<SqlInfo>>();
085        private TSQLEnv sqlenv = null;
086        private ModelBindingManager modelManager = new ModelBindingManager();
087        private ModelFactory modelFactory = new ModelFactory(modelManager);
088        private PipelinedFunctionAnalyzer pipelinedAnalyzer;
089        private List<Long> tableIds = new ArrayList<Long>();
090        private TTableList hiveFromTables;
091        private dataflow dataflow;
092        private String dataflowString;
093        private Option option = new Option();
094
095        {
096                modelManager.TABLE_COLUMN_ID = option.getStartId();
097                modelManager.RELATION_ID = option.getStartId();
098                ModelBindingManager.set(modelManager);
099                ModelBindingManager.setGlobalStmtStack(stmtStack);
100                ModelBindingManager.setGlobalOption(option);
101                ModelBindingManager.setGlobalSqlInfo(sqlInfoMap);
102        }
103
104    private void initPipelinedFunctionAnalyzer() {
105        // Only initialize pipelinedAnalyzer for Oracle
106        if (option.getVendor() == EDbVendor.dbvoracle)
107        {
108            pipelinedAnalyzer = new PipelinedFunctionAnalyzer(modelManager, modelFactory, option);
109        }
110        else{
111            pipelinedAnalyzer = null;
112        }
113    }
114
115        public DataFlowAnalyzer(String sqlContent, Option option) {
116                SqlInfo[] sqlInfos = new SqlInfo[1];
117                SqlInfo info = new SqlInfo();
118                info.setSql(sqlContent);
119                info.setOriginIndex(0);
120                sqlInfos[0] = info;
121                this.option = option;
122                ModelBindingManager.setGlobalOption(this.option);
123                this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
124    }
125
126    public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
127                        String defaultDatabase, String defaltSchema) {
128                SqlInfo[] sqlInfos = new SqlInfo[1];
129                SqlInfo info = new SqlInfo();
130                info.setSql(sqlContent);
131                info.setOriginIndex(0);
132                sqlInfos[0] = info;
133                option.setVendor(dbVendor);
134                option.setSimpleOutput(simpleOutput);
135                option.setDefaultServer(defaultServer);
136                option.setDefaultDatabase(defaultDatabase);
137                option.setDefaultSchema(defaltSchema);
138                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
139        }
140
141        public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput) {
142                SqlInfo[] sqlInfos = new SqlInfo[1];
143                SqlInfo info = new SqlInfo();
144                info.setSql(sqlContent);
145                info.setOriginIndex(0);
146                sqlInfos[0] = info;
147                option.setVendor(dbVendor);
148                option.setSimpleOutput(simpleOutput);
149                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
150    }
151
152        public DataFlowAnalyzer(String[] sqlContents, Option option) {
153                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
154                for (int i = 0; i < sqlContents.length; i++) {
155                        SqlInfo info = new SqlInfo();
156                        info.setSql(sqlContents[i]);
157                        info.setOriginIndex(0);
158                        sqlInfos[i] = info;
159                }
160                this.option = option;
161                ModelBindingManager.setGlobalOption(this.option);
162                this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
163        }
164
165        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
166                        String defaultDatabase, String defaltSchema) {
167                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
168                for (int i = 0; i < sqlContents.length; i++) {
169                        SqlInfo info = new SqlInfo();
170                        info.setSql(sqlContents[i]);
171                        info.setOriginIndex(0);
172                        sqlInfos[i] = info;
173                }
174                option.setVendor(dbVendor);
175                option.setSimpleOutput(simpleOutput);
176                option.setDefaultServer(defaultServer);
177                option.setDefaultDatabase(defaultDatabase);
178                option.setDefaultSchema(defaltSchema);
179                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
180        }
181
182        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput) {
183                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
184                for (int i = 0; i < sqlContents.length; i++) {
185                        SqlInfo info = new SqlInfo();
186                        info.setSql(sqlContents[i]);
187                        info.setOriginIndex(0);
188                        sqlInfos[i] = info;
189                }
190                option.setVendor(dbVendor);
191                option.setSimpleOutput(simpleOutput);
192                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
193        }
194
195        public DataFlowAnalyzer(SqlInfo[] sqlInfos, Option option) {
196                this.sqlInfos = sqlInfos;
197                this.option = option;
198                ModelBindingManager.setGlobalOption(this.option);
199        }
200
201        public DataFlowAnalyzer(SqlInfo[] sqlInfos, EDbVendor dbVendor, boolean simpleOutput) {
202                this.sqlInfos = sqlInfos;
203                option.setVendor(dbVendor);
204                option.setSimpleOutput(simpleOutput);
205        }
206
207        public DataFlowAnalyzer(File[] sqlFiles, Option option) {
208                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
209                for (int i = 0; i < sqlFiles.length; i++) {
210                        SqlInfo info = new SqlInfo();
211                        info.setSql(SQLUtil.getFileContent(sqlFiles[i]));
212                        info.setFileName(sqlFiles[i].getName());
213                        info.setFilePath(sqlFiles[i].getAbsolutePath());
214                        info.setOriginIndex(0);
215                        sqlInfos[i] = info;
216                }
217                this.sqlInfos = sqlInfos;
218                this.option = option;
219                ModelBindingManager.setGlobalOption(this.option);
220        }
221
222        public DataFlowAnalyzer(File[] sqlFiles, EDbVendor dbVendor, boolean simpleOutput) {
223                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
224                for (int i = 0; i < sqlFiles.length; i++) {
225                        SqlInfo info = new SqlInfo();
226                        info.setSql(SQLUtil.getFileContent(sqlFiles[i]));
227                        info.setFileName(sqlFiles[i].getName());
228                        info.setFilePath(sqlFiles[i].getAbsolutePath());
229                        info.setOriginIndex(0);
230                        sqlInfos[i] = info;
231                }
232                this.sqlInfos = sqlInfos;
233                option.setVendor(dbVendor);
234                option.setSimpleOutput(simpleOutput);
235        }
236
237        public DataFlowAnalyzer(File sqlFile, Option option) {
238                File[] children = SQLUtil.listFiles(sqlFile);
239                SqlInfo[] sqlInfos = new SqlInfo[children.length];
240                for (int i = 0; i < children.length; i++) {
241                        SqlInfo info = new SqlInfo();
242                        info.setSql(SQLUtil.getFileContent(children[i]));
243                        info.setFileName(children[i].getName());
244                        info.setFilePath(children[i].getAbsolutePath());
245                        info.setOriginIndex(0);
246                        sqlInfos[i] = info;
247                }
248                this.sqlInfos = sqlInfos;
249                this.option = option;
250                ModelBindingManager.setGlobalOption(this.option);
251        }
252
253        public DataFlowAnalyzer(File sqlFile, EDbVendor dbVendor, boolean simpleOutput) {
254                File[] children = SQLUtil.listFiles(sqlFile);
255                List<SqlInfo> sqlInfos = new ArrayList<>();
256                for (int i = 0; i < children.length; i++) {
257                        SqlInfo info = new SqlInfo();
258                        info.setSql(SQLUtil.getFileContent(children[i]));
259            if(children[i].getName().toLowerCase().endsWith(".csv")){
260                if(!MetadataReader.isMetadata(info.getSql())){
261                    continue;
262                }
263            }
264            else if(children[i].getName().toLowerCase().endsWith(".json")){
265                if (!(MetadataReader.isGrabit(info.getSql()) || MetadataReader.isSqlflow(info.getSql()))){
266                    continue;
267                }
268            }
269                        info.setFileName(children[i].getName());
270                        info.setFilePath(children[i].getAbsolutePath());
271                        info.setOriginIndex(0);
272                        sqlInfos.add(info);
273                }
274                this.sqlInfos = sqlInfos.toArray(new SqlInfo[0]);
275                option.setVendor(dbVendor);
276                option.setSimpleOutput(simpleOutput);
277        }
278
279        public boolean isIgnoreRecordSet() {
280                return option.isIgnoreRecordSet();
281        }
282
283        public void setIgnoreRecordSet(boolean ignoreRecordSet) {
284                option.setIgnoreRecordSet(ignoreRecordSet);
285        }
286
287        public boolean isSimpleShowTopSelectResultSet() {
288                return option.isSimpleShowTopSelectResultSet();
289        }
290
291        public void setSimpleShowTopSelectResultSet(boolean simpleShowTopSelectResultSet) {
292                option.setSimpleShowTopSelectResultSet(simpleShowTopSelectResultSet);
293        }
294
295        public boolean isSimpleShowFunction() {
296                return option.isSimpleShowFunction();
297        }
298
299        public void setSimpleShowFunction(boolean simpleShowFunction) {
300                option.setSimpleShowFunction(simpleShowFunction);
301        }
302
303        public boolean isShowJoin() {
304                return option.isShowJoin();
305        }
306
307        public void setShowJoin(boolean showJoin) {
308                option.setShowJoin(showJoin);
309        }
310
311        public void setShowCallRelation(boolean showCallRelation) {
312                option.setShowCallRelation(showCallRelation);
313        }
314
315        public boolean isShowCallRelation() {
316                return option.isShowCallRelation();
317        }
318
319        public boolean isShowImplicitSchema() {
320                return option.isShowImplicitSchema();
321        }
322
323        public void setShowImplicitSchema(boolean showImplicitSchema) {
324                option.setShowImplicitSchema(showImplicitSchema);
325        }
326
327        public boolean isShowConstantTable() {
328                return option.isShowConstantTable();
329        }
330
331        public void setShowConstantTable(boolean showConstantTable) {
332                option.setShowConstantTable(showConstantTable);
333        }
334
335        public boolean isShowCountTableColumn() {
336                return option.isShowCountTableColumn();
337        }
338
339        public void setShowCountTableColumn(boolean showCountTableColumn) {
340                option.setShowCountTableColumn(showCountTableColumn);
341        }
342
343        public boolean isTransform() {
344                return option.isTransform();
345        }
346
347        public void setTransform(boolean transform) {
348                option.setTransform(transform);
349                if (option.isTransformCoordinate()) {
350                        option.setTransform(true);
351                }
352        }
353
354        public boolean isTransformCoordinate() {
355                return option.isTransformCoordinate();
356        }
357
358        public void setTransformCoordinate(boolean transformCoordinate) {
359                option.setTransformCoordinate(transformCoordinate);
360                if (transformCoordinate) {
361                        option.setTransform(true);
362                }
363        }
364
365        public boolean isLinkOrphanColumnToFirstTable() {
366                return option.isLinkOrphanColumnToFirstTable();
367        }
368
369        public void setLinkOrphanColumnToFirstTable(boolean linkOrphanColumnToFirstTable) {
370                option.setLinkOrphanColumnToFirstTable(linkOrphanColumnToFirstTable);
371        }
372
373        public boolean isIgnoreTemporaryTable() {
374                return option.isIgnoreTemporaryTable();
375        }
376
377        public void setIgnoreTemporaryTable(boolean ignoreTemporaryTable) {
378                option.setIgnoreTemporaryTable(ignoreTemporaryTable);
379        }
380        
381        public boolean isIgnoreCoordinate() {
382                return option.isIgnoreCoordinate();
383        }
384
385        public void setIgnoreCoordinate(boolean ignoreCoordinate) {
386                option.setIgnoreCoordinate(ignoreCoordinate);
387        }
388
389        public void setHandleListener(DataFlowHandleListener listener) {
390                option.setHandleListener(listener);
391        }
392
393        public void setSqlEnv(TSQLEnv sqlenv) {
394                this.sqlenv = sqlenv;
395        }
396
397        public void setOption(Option option) {
398                this.option = option;
399                ModelBindingManager.setGlobalOption(this.option);
400        }
401
402        public Option getOption() {
403                return option;
404        }
405
406        public synchronized String chechSyntax() {
407                StringBuilder builder = new StringBuilder();
408                if (sqlInfos != null) {
409                        for (SqlInfo sqlInfo : sqlInfos) {
410                                String content = sqlInfo.getSql();
411                                if (content != null && content.indexOf("<dlineage") != -1) {
412                                        try {
413                                                XML2Model.loadXML(dataflow.class, content);
414                                                continue;
415                                        } catch (Exception e) {
416                                                builder.append("Parsing dataflow ").append("occurs errors.\n").append(e.getMessage())
417                                                                .append("\n");
418                                        }
419                                }
420                                if (content != null && content.trim().startsWith("{")) {
421                                        Map queryObject = (Map) JSON.parseObject(content);
422                                        content = (String) queryObject.get("sourceCode");
423                                }
424                                if (MetadataReader.isMetadata(content)) {
425                                        continue;
426                                }
427                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
428                                sqlparser.sqltext = content;
429                                int result = sqlparser.parse();
430                                if (result != 0) {
431                                        builder.append("Parsing sql ").append("occurs errors.\n").append(sqlparser.getErrormessage())
432                                                        .append("\n");
433                                }
434                        }
435                }
436                return builder.toString();
437        }
438
439        public synchronized String generateDataFlow(boolean withExtraInfo) {
440                if (ModelBindingManager.get() == null) {
441                        ModelBindingManager.set(modelManager);
442                }
443
444        initPipelinedFunctionAnalyzer();
445
446                dataflow = analyzeSqlScript();
447
448                if (dataflow != null && !withExtraInfo && dataflow.getResultsets() != null) {
449                        for (table t : dataflow.getResultsets()) {
450                                t.setIsTarget(null);
451                                if (t.getColumns() != null) {
452                                        for (column t1 : t.getColumns()) {
453                                                t1.setIsFunction(null);
454                                        }
455                                }
456                        }
457                }
458                
459                if(dataflow!=null && dataflow.getRelationships()!=null && option.getFilterRelationTypes()!=null && !option.getFilterRelationTypes().isEmpty()) {
460                        List<relationship> relationships = new ArrayList<relationship>();
461                        for(relationship relationship: dataflow.getRelationships()) {
462                                if(option.getFilterRelationTypes().contains(relationship.getType())) {
463                                        relationships.add(relationship);
464                                }
465                        }
466                        dataflow.setRelationships(relationships);
467                }
468
469                if (option.getHandleListener() != null) {
470                        option.getHandleListener().endAnalyze(dataflow);
471                }
472
473                if (option.isOutput()) {
474                        if (option.getHandleListener() != null) {
475                                option.getHandleListener().startOutputDataFlowXML();
476                        }
477                        if (dataflow != null) {
478                                if (option.isTextFormat()) {
479                                        dataflowString = getTextOutput(dataflow);
480                                } else {
481                                        try {
482                                                dataflowString = XML2Model.saveXML(dataflow);
483                                        }catch (Exception e){
484                                                logger.error("Output dataflow to xml failed.", e);
485                                                dataflowString = null;
486                                        }
487                                }
488                        }
489                        if (option.getHandleListener() != null) {
490                                option.getHandleListener().endOutputDataFlowXML(dataflowString == null ? 0 : dataflowString.length());
491                        }
492                }
493
494                return dataflowString;
495        }
496
497        private dataflow removeDuplicateColumns(dataflow dataflow) {
498                List<table> tables = new ArrayList<table>();
499                if (dataflow.getTables() != null) {
500                        tables.addAll(dataflow.getTables());
501                }
502                if (dataflow.getViews() != null) {
503                        tables.addAll(dataflow.getViews());
504                }
505                if (dataflow.getStages() != null) {
506                        tables.addAll(dataflow.getStages());
507                }
508                if (dataflow.getDatasources() != null) {
509                        tables.addAll(dataflow.getDatasources());
510                }
511                if (dataflow.getStreams() != null) {
512                        tables.addAll(dataflow.getStreams());
513                }
514                if (dataflow.getPaths() != null) {
515                        tables.addAll(dataflow.getPaths());
516                }
517                if (dataflow.getVariables() != null) {
518                        tables.addAll(dataflow.getVariables());
519                }
520                if (dataflow.getResultsets() != null) {
521                        tables.addAll(dataflow.getResultsets());
522                }
523                for (table table : tables) {
524                        if (table.getColumns() == null) {
525                                continue;
526                        }
527                        Set<String> columnIds = new HashSet<String>();
528                        Iterator<column> iter = table.getColumns().iterator();
529                        while(iter.hasNext()) {
530                                column column = iter.next();
531                                String id = column.getId();
532                                if (columnIds.contains(id)) {
533                                        iter.remove();
534                                } else {
535                                        columnIds.add(id);
536                                }
537                        }
538                }
539                return dataflow;
540        }
541
542        public synchronized String generateDataFlow() {
543                return generateDataFlow(false);
544        }
545
546        public synchronized String generateSqlInfos() {
547                return JSON.toJSONString(sqlInfoMap);
548        }
549
550        public Map<String, List<SqlInfo>> getSqlInfos() {
551                return sqlInfoMap;
552        }
553
554        public Map getHashSQLMap() {
555                return modelManager.getHashSQLMap();
556        }
557        
558        public Map getDynamicSQLMap() {
559                return modelManager.getDynamicSQLMap();
560        }
561
562        /**
563         * @deprecated please use SqlInfoHelper.getSelectedDbObjectInfo
564         */
565        public DbObjectPosition getSelectedDbObjectInfo(Coordinate start, Coordinate end) {
566                if (start == null || end == null) {
567                        throw new IllegalArgumentException("Coordinate can't be null.");
568                }
569
570                String hashCode = start.getHashCode();
571
572                if (hashCode == null) {
573                        throw new IllegalArgumentException("Coordinate hashcode can't be null.");
574                }
575
576                int dbObjectStartLine = (int) start.getX() - 1;
577                int dbObjectStarColumn = (int) start.getY() - 1;
578                int dbObjectEndLine = (int) end.getX() - 1;
579                int dbObjectEndColumn = (int) end.getY() - 1;
580                List<SqlInfo> sqlInfoList;
581                if (hashCode.matches("\\d+")) {
582                        sqlInfoList = sqlInfoMap.getValueAtIndex(Integer.valueOf(hashCode));
583                } else {
584                        sqlInfoList = sqlInfoMap.get(hashCode);
585                }
586                for (int j = 0; j < sqlInfoList.size(); j++) {
587                        SqlInfo sqlInfo = sqlInfoList.get(j);
588                        int startLine = sqlInfo.getLineStart();
589                        int endLine = sqlInfo.getLineEnd();
590                        if (dbObjectStartLine >= startLine && dbObjectStartLine <= endLine) {
591                                DbObjectPosition position = new DbObjectPosition();
592                                position.setFile(sqlInfo.getFileName());
593                                position.setFilePath(sqlInfo.getFilePath());
594                                position.setSql(sqlInfo.getSql());
595                                position.setIndex(sqlInfo.getOriginIndex());
596                                List<Pair<Integer, Integer>> positions = position.getPositions();
597                                positions.add(new Pair<Integer, Integer>(
598                                                dbObjectStartLine - startLine + sqlInfo.getOriginLineStart() + 1, dbObjectStarColumn + 1));
599                                positions.add(new Pair<Integer, Integer>(dbObjectEndLine - startLine + sqlInfo.getOriginLineStart() + 1,
600                                                dbObjectEndColumn + 1));
601                                return position;
602                        }
603                }
604                return null;
605        }
606
607        public static dataflow mergeTables(dataflow dataflow, Long startId) {
608                return mergeTables(dataflow, startId, new Option());
609        }
610
611        public static dataflow mergeTables(dataflow dataflow, Long startId, Option option) {
612                List<table> tableCopy = new ArrayList<table>();
613                List<table> viewCopy = new ArrayList<table>();
614                List<table> databaseCopy = new ArrayList<table>();
615                List<table> schemaCopy = new ArrayList<table>();
616                List<table> stageCopy = new ArrayList<table>();
617                List<table> dataSourceCopy = new ArrayList<table>();
618                List<table> streamCopy = new ArrayList<table>();
619                List<table> fileCopy = new ArrayList<table>();
620                List<table> variableCopy = new ArrayList<table>();
621                List<table> cursorCopy = new ArrayList<table>();
622                List<table> resultSetCopy = new ArrayList<table>();
623                if (dataflow.getTables() != null) {
624                        tableCopy.addAll(dataflow.getTables());
625                }
626                dataflow.setTables(tableCopy);
627                if (dataflow.getViews() != null) {
628                        viewCopy.addAll(dataflow.getViews());
629                }
630                dataflow.setViews(viewCopy);
631                if (dataflow.getDatabases() != null) {
632                        databaseCopy.addAll(dataflow.getDatabases());
633                }
634                dataflow.setDatabases(databaseCopy);
635                if (dataflow.getSchemas() != null) {
636                        schemaCopy.addAll(dataflow.getSchemas());
637                }
638                dataflow.setSchemas(schemaCopy);
639                if (dataflow.getStages() != null) {
640                        stageCopy.addAll(dataflow.getStages());
641                }
642                dataflow.setStages(stageCopy);
643                if (dataflow.getDatasources() != null) {
644                        dataSourceCopy.addAll(dataflow.getDatasources());
645                }
646                dataflow.setDatasources(dataSourceCopy);
647                if (dataflow.getStreams() != null) {
648                        streamCopy.addAll(dataflow.getStreams());
649                }
650                dataflow.setStreams(streamCopy);
651                if (dataflow.getPaths() != null) {
652                        fileCopy.addAll(dataflow.getPaths());
653                }
654                dataflow.setPaths(fileCopy);
655                if (dataflow.getVariables() != null) {
656                        variableCopy.addAll(dataflow.getVariables());
657                }
658                dataflow.setVariables(variableCopy);
659                if (dataflow.getResultsets() != null) {
660                        resultSetCopy.addAll(dataflow.getResultsets());
661                }
662                dataflow.setResultsets(resultSetCopy);
663
664                Map<String, List<table>> tableMap = new HashMap<String, List<table>>();
665                Map<String, String> tableTypeMap = new HashMap<String, String>();
666                Map<String, TMssqlCreateType> mssqlTypeMap = new HashMap<String, TMssqlCreateType>();
667                Map<String, String> tableIdMap = new HashMap<String, String>();
668
669                Map<String, List<column>> columnMap = new HashMap<String, List<column>>();
670                Map<String, Set<String>> tableColumnMap = new HashMap<String, Set<String>>();
671                Map<String, String> columnIdMap = new HashMap<String, String>();
672                Map<String, column> columnMergeIdMap = new HashMap<String, column>();
673
674                List<table> tables = new ArrayList<table>();
675                tables.addAll(dataflow.getTables());
676                tables.addAll(dataflow.getViews());
677                tables.addAll(dataflow.getDatabases());
678                tables.addAll(dataflow.getSchemas());
679                tables.addAll(dataflow.getStages());
680                tables.addAll(dataflow.getDatasources());
681                tables.addAll(dataflow.getStreams());
682                tables.addAll(dataflow.getPaths());
683                tables.addAll(dataflow.getResultsets());
684                tables.addAll(dataflow.getVariables());
685                
686                Set<String> columnIds = new HashSet<String>();
687
688                for (table table : tables) {
689                        String qualifiedTableName = DlineageUtil.getQualifiedTableName(table);
690                        String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName);
691
692                        if (!tableMap.containsKey(tableFullName)) {
693                                tableMap.put(tableFullName, new ArrayList<table>());
694                        }
695
696                        tableMap.get(tableFullName).add(table);
697
698                        if (!tableTypeMap.containsKey(tableFullName)) {
699                                tableTypeMap.put(tableFullName, table.getType());
700                        } else if ("view".equals(table.getSubType())) {
701                                tableTypeMap.put(tableFullName, table.getType());
702                        } else if ("database".equals(table.getSubType())) {
703                                tableTypeMap.put(tableFullName, table.getType());
704                        } else if ("schema".equals(table.getSubType())) {
705                                tableTypeMap.put(tableFullName, table.getType());
706                        } else if ("stage".equals(table.getSubType())) {
707                                tableTypeMap.put(tableFullName, table.getType());
708                        } else if ("sequence".equals(table.getSubType())) {
709                                tableTypeMap.put(tableFullName, table.getType());
710                        } else if ("datasource".equals(table.getSubType())) {
711                                tableTypeMap.put(tableFullName, table.getType());
712                        } else if ("stream".equals(table.getSubType())) {
713                                tableTypeMap.put(tableFullName, table.getType());
714                        } else if ("file".equals(table.getSubType())) {
715                                tableTypeMap.put(tableFullName, table.getType());
716                        } else if ("table".equals(tableTypeMap.get(tableFullName))) {
717                                tableTypeMap.put(tableFullName, table.getType());
718                        } else if ("variable".equals(tableTypeMap.get(tableFullName))) {
719                                tableTypeMap.put(tableFullName, table.getType());
720                        }
721
722                        if (table.getColumns() != null) {
723                                if (!tableColumnMap.containsKey(tableFullName)) {
724                                        tableColumnMap.put(tableFullName, new LinkedHashSet<String>());
725                                }
726                                for (column column : table.getColumns()) {
727                                        String columnFullName = tableFullName + "."
728                                                        + (column.getQualifiedTable() != null
729                                                                        ? (DlineageUtil.getIdentifierNormalTableName(column.getQualifiedTable()) + ".")
730                                                                        : "")
731                                                        + ("false".equals(table.getIsTarget()) ? DlineageUtil.normalizeColumnName(column.getName())
732                                                                        : DlineageUtil.getIdentifierNormalColumnName(column.getName()));
733
734                                        if (!columnMap.containsKey(columnFullName)) {
735                                                columnMap.put(columnFullName, new ArrayList<column>());
736                                                tableColumnMap.get(tableFullName).add(columnFullName);
737                                        }
738
739                                        columnMap.get(columnFullName).add(column);
740                                        columnIds.add(column.getId());
741                                }
742                        }
743                }
744
745                Iterator<String> tableNameIter = tableMap.keySet().iterator();
746                while (tableNameIter.hasNext()) {
747                        String tableName = tableNameIter.next();
748                        List<table> tableList = tableMap.get(tableName);
749                        table table;
750                        if (tableList.size() > 1) {
751                                table standardTable = tableList.get(0);
752                                // Function允许重名，不做合并处理
753                                if (standardTable.isFunction()) {
754                                        continue;
755                                }
756                                
757                                // Variable允许重名，不做合并处理
758                                if (standardTable.isVariable()) {
759                                        continue;
760                                }
761                                
762                                // 临时表不做合并处理
763                                if(SQLUtil.isTempTable(standardTable)) {
764                                        continue;
765                                }
766
767                                String type = tableTypeMap.get(tableName);
768                                table = new table();
769                                table.setId(String.valueOf(++startId));
770                                table.setServer(standardTable.getServer());
771                                table.setDatabase(standardTable.getDatabase());
772                                table.setSchema(standardTable.getSchema());
773                                table.setName(standardTable.getName());
774                                table.setDisplayName(standardTable.getDisplayName());
775                                table.setParent(standardTable.getParent());
776                                table.setMore(standardTable.getMore());
777                                table.setColumns(new ArrayList<column>());
778                                table.setSubType(standardTable.getSubType());
779                                Set<String> processIds = new LinkedHashSet<String>();
780                                for (int k = 0; k < tableList.size(); k++) {
781                                        if (tableList.get(k).getProcessIds() != null) {
782                                                processIds.addAll(tableList.get(k).getProcessIds());
783                                        }
784                                }
785                                if (!processIds.isEmpty()) {
786                                        table.setProcessIds(new ArrayList<String>(processIds));
787                                }
788                                Set<String> alias = new LinkedHashSet<String>();
789                                for (int k = 0; k < tableList.size(); k++) {
790                                        if (tableList.get(k).getAlias() != null) {
791                                                alias.addAll(Arrays.asList(tableList.get(k).getAlias().split("\\s*,\\s*")));
792                                        }
793                                }
794                                if (!alias.isEmpty()) {
795                                        String aliasString = Arrays.toString(alias.toArray(new String[0]));
796                                        table.setAlias(aliasString.substring(1, aliasString.length() - 1));
797                                }
798                                table.setType(type);
799                                for (table item : tableList) {
800                                        if (!SQLUtil.isEmpty(table.getCoordinate()) && !SQLUtil.isEmpty(item.getCoordinate())) {
801                                                if (table.getCoordinate().indexOf(item.getCoordinate()) == -1) {
802                                                        table.appendCoordinate(item.getCoordinate());
803                                                }
804                                        } else if (!SQLUtil.isEmpty(item.getCoordinate())) {
805                                                table.setCoordinate(item.getCoordinate());
806                                        }
807
808                                        if (item.getStarStmt() != null) {
809                                                table.setStarStmt(item.getStarStmt());
810                                        }
811
812                                        tableIdMap.put(item.getId(), table.getId());
813
814                                        if (item.isView()) {
815                                                dataflow.getViews().remove(item);
816                                        } else if (item.isDatabaseType()) {
817                                                dataflow.getDatabases().remove(item);
818                                        } else if (item.isSchemaType()) {
819                                                dataflow.getSchemas().remove(item);
820                                        } else if (item.isStage()) {
821                                                dataflow.getStages().remove(item);
822                                        } else if (item.isDataSource()) {
823                                                dataflow.getDatasources().remove(item);
824                                        } else if (item.isStream()) {
825                                                dataflow.getStreams().remove(item);
826                                        } else if (item.isFile()) {
827                                                dataflow.getPaths().remove(item);
828                                        } else if (item.isVariable()) {
829                                                dataflow.getVariables().remove(item);
830                                        } else if (item.isTable()) {
831                                                dataflow.getTables().remove(item);
832                                        } else if (item.isResultSet()) {
833                                                dataflow.getResultsets().remove(item);
834                                        }
835                                }
836
837                                if (table.isView()) {
838                                        dataflow.getViews().add(table);
839                                } else if (table.isDatabaseType()) {
840                                        dataflow.getDatabases().add(table);
841                                } else if (table.isSchemaType()) {
842                                        dataflow.getSchemas().add(table);
843                                } else if (table.isStage()) {
844                                        dataflow.getStages().add(table);
845                                } else if (table.isDataSource()) {
846                                        dataflow.getDatasources().add(table);
847                                } else if (table.isStream()) {
848                                        dataflow.getStreams().add(table);
849                                } else if (table.isFile()) {
850                                        dataflow.getPaths().add(table);
851                                } else if (table.isVariable()) {
852                                        dataflow.getVariables().add(table);
853                                } else if (table.isResultSet()) {
854                                        dataflow.getResultsets().add(table);
855                                } else {
856                                        dataflow.getTables().add(table);
857                                }
858                        } else {
859                                table = tableList.get(0);
860                                if(Boolean.TRUE.toString().equals(table.getIsDetermined())){
861                                        continue;
862                                }
863                                
864                                if (option.isIgnoreUnusedSynonym() && SubType.synonym.name().equals(table.getSubType())) {
865                                        dataflow.getTables().remove(table);
866                                        tableColumnMap.get(tableName).clear();
867                                        continue;
868                                }
869                        }
870
871                        Set<String> columns = tableColumnMap.get(tableName);
872                        Iterator<String> columnIter = columns.iterator();
873                        List<column> mergeColumns = new ArrayList<column>();
874                        while (columnIter.hasNext()) {
875                                String columnName = columnIter.next();
876                                List<column> columnList = columnMap.get(columnName);
877                                List<column> functions = new ArrayList<column>();
878                                for (column t : columnList) {
879                                        if (Boolean.TRUE.toString().equals(t.getIsFunction())) {
880                                                functions.add(t);
881                                        }
882                                }
883                                if (functions != null && !functions.isEmpty()) {
884                                        for (column function : functions) {
885                                                mergeColumns.add(function);
886                                                columnIdMap.put(function.getId(), function.getId());
887                                                columnMergeIdMap.put(function.getId(), function);
888                                        }
889
890                                        columnList.removeAll(functions);
891                                }
892                                if (!columnList.isEmpty()) {
893                                        column firstColumn = columnList.iterator().next();
894                                        if (columnList.size() > 1) {
895                                                column mergeColumn = new column();
896                                                mergeColumn.setId(String.valueOf(++startId));
897                                                mergeColumn.setName(firstColumn.getName());
898                                                mergeColumn.setDisplayName(firstColumn.getDisplayName());
899                                                mergeColumn.setSource(firstColumn.getSource());
900                                                mergeColumn.setQualifiedTable(firstColumn.getQualifiedTable());
901                                                mergeColumn.setDataType(firstColumn.getDataType());
902                                                mergeColumn.setForeignKey(firstColumn.isForeignKey());
903                                                mergeColumn.setPrimaryKey(firstColumn.isPrimaryKey());
904                                                mergeColumn.setUnqiueKey(firstColumn.isUnqiueKey());
905                                                mergeColumn.setIndexKey(firstColumn.isIndexKey());
906                                                mergeColumns.add(mergeColumn);
907                                                for (column item : columnList) {
908                                                        mergeColumn.appendCoordinate(item.getCoordinate());
909                                                        columnIdMap.put(item.getId(), mergeColumn.getId());
910                                                }
911                                                columnMergeIdMap.put(mergeColumn.getId(), mergeColumn);
912                                                columnIds.add(mergeColumn.getId());
913                                        } else {
914                                                mergeColumns.add(firstColumn);
915                                                columnIdMap.put(firstColumn.getId(), firstColumn.getId());
916                                                columnMergeIdMap.put(firstColumn.getId(), firstColumn);
917                                        }
918                                }
919                        }
920                        table.setColumns(mergeColumns);
921                }
922
923                if (dataflow.getRelationships() != null) {
924                        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
925                        for (int i = 0; i < dataflow.getRelationships().size(); i++) {
926                                relationship relation = dataflow.getRelationships().get(i);
927                                
928                                if("crud".equals(relation.getType()) && option.getAnalyzeMode() == AnalyzeMode.crud) {
929                                        String jsonString = JSON.toJSONString(relation, true);
930                                        String key = SHA256.getMd5(jsonString);
931                                        if (!mergeRelations.containsKey(key)) {
932                                                mergeRelations.put(key, relation);
933                                        }
934                                        continue;
935                                }
936                                
937                                targetColumn target = relation.getTarget();
938                                if ("call".equals(relation.getType())) {
939                                        target = relation.getCaller();
940                                }
941                                if (target != null && tableIdMap.containsKey(target.getParent_id())) {
942                                        target.setParent_id(tableIdMap.get(target.getParent_id()));
943                                }
944
945                                if (columnIdMap.containsKey(target.getId())) {
946                                        target.setId(columnIdMap.get(target.getId()));
947                                        target.setCoordinate(columnMergeIdMap.get(target.getId()).getCoordinate());
948                                }
949                                else if(option.isIgnoreUnusedSynonym() && EffectType.synonym.name().equals(relation.getEffectType())){
950                                        continue;
951                                }
952                                
953                                if (!"call".equals(relation.getType()) && !columnIds.contains(target.getId())) {
954                                        continue;
955                                }
956
957                                List<sourceColumn> sources = relation.getSources();
958                                if ("call".equals(relation.getType())) {
959                                        sources = relation.getCallees();
960                                }
961                                Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
962                                if (sources != null) {
963                                        for (sourceColumn source : sources) {
964                                                if (!"call".equals(relation.getType()) && !columnIds.contains(source.getId())) {
965                                                        continue;
966                                                }
967                                                if (tableIdMap.containsKey(source.getParent_id())) {
968                                                        source.setParent_id(tableIdMap.get(source.getParent_id()));
969                                                }
970                                                if (tableIdMap.containsKey(source.getSource_id())) {
971                                                        source.setSource_id(tableIdMap.get(source.getSource_id()));
972                                                }
973                                                if (columnIdMap.containsKey(source.getId())) {
974                                                        source.setId(columnIdMap.get(source.getId()));
975                                                        source.setCoordinate(columnMergeIdMap.get(source.getId()).getCoordinate());
976                                                }
977                                        }
978
979                                        sourceSet.addAll(sources);
980                                        if ("call".equals(relation.getType())) {
981                                                relation.setCallees(new ArrayList<sourceColumn>(sourceSet));
982                                        } else {
983                                                relation.setSources(new ArrayList<sourceColumn>(sourceSet));
984                                        }
985                                }
986
987                                String jsonString = JSON.toJSONString(relation, true);
988                String key = SHA256.getMd5(jsonString);
989                                if (!mergeRelations.containsKey(key)) {
990                                        mergeRelations.put(key, relation);
991                                }
992                        }
993
994                        dataflow.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
995                }
996
997                tableMap.clear();
998                tableTypeMap.clear();
999                tableIdMap.clear();
1000                columnMap.clear();
1001                tableColumnMap.clear();
1002                columnIdMap.clear();
1003                columnMergeIdMap.clear();
1004                tables.clear();
1005                return dataflow;
1006        }
1007
1008        public synchronized dataflow getDataFlow() {
1009                if (dataflow != null) {
1010                        return dataflow;
1011                } else if (dataflowString != null) {
1012                        return XML2Model.loadXML(dataflow.class, dataflowString);
1013                }
1014                return null;
1015        }
1016
1017        List<ErrorInfo> metadataErrors = new ArrayList<>();
1018        
1019        private synchronized dataflow analyzeSqlScript() {
1020                init();
1021
1022                try {
1023                        dataflow dataflow = new dataflow();
1024                        
1025                        if (sqlInfos != null) {
1026                                if (option.getHandleListener() != null) {
1027                                        if (sqlInfos.length == 1) {
1028                                                option.getHandleListener().startAnalyze(null, sqlInfos[0].getSql().length(), false);
1029                                        } else {
1030                                                option.getHandleListener().startAnalyze(null, sqlInfos.length, true);
1031                                        }
1032                                }
1033
1034                                if (sqlenv == null) {
1035                                        if (option.getHandleListener() != null) {
1036                                                option.getHandleListener().startParseSQLEnv();
1037                                        }
1038                                        TSQLEnv[] sqlenvs = new SQLEnvParser(option.getDefaultServer(), option.getDefaultDatabase(),
1039                                                        option.getDefaultSchema()).parseSQLEnv(option.getVendor(), sqlInfos);
1040                                        if (sqlenvs != null && sqlenvs.length > 0) {
1041                                                sqlenv = sqlenvs[0];
1042                                        }
1043                                        if (option.getHandleListener() != null) {
1044                                                option.getHandleListener().endParseSQLEnv();
1045                                        }
1046                                }
1047                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
1048                                Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap = new LinkedHashMap<String, Pair3<StringBuilder, AtomicInteger, String>>();
1049                                for (int i = 0; i < sqlInfos.length; i++) {
1050                                        SqlInfo sqlInfo = sqlInfos[i];
1051                                        if (sqlInfo == null) {
1052                                                sqlInfoMap.put(String.valueOf(i), new ArrayList<SqlInfo>());
1053                                                continue;
1054                                        }
1055                                        String sql = sqlInfo.getSql();
1056                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFileName() != null
1057                                                        && new File(sqlInfo.getFileName()).exists()) {
1058                                                sql = SQLUtil.getFileContent(sqlInfo.getFileName());
1059                                        }
1060                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFilePath() != null
1061                                                        && new File(sqlInfo.getFilePath()).exists()) {
1062                                                sql = SQLUtil.getFileContent(sqlInfo.getFilePath());
1063                                        }
1064                                        String sqlTrim = null;
1065                                        if (sql != null) {
1066                                                sqlTrim = sql.substring(0, Math.min(sql.length(), 512)).trim();
1067                                        }
1068                                        if(sql!=null && sqlTrim.startsWith("<") && sqlTrim.indexOf("<dlineage")!=-1){
1069                                                dataflow temp = XML2Model.loadXML(dataflow.class, sql);
1070                                                if(sqlInfos.length == 1){
1071                                                        dataflow = temp;
1072                                                }
1073                                                else {
1074                                                        if (temp.getTables() != null) {
1075                                                                dataflow.getTables().addAll(temp.getTables());
1076                                                        }
1077                                                        if (temp.getViews() != null) {
1078                                                                dataflow.getViews().addAll(temp.getViews());
1079                                                        }
1080                                                        if (temp.getResultsets() != null) {
1081                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1082                                                        }
1083                                                        if (temp.getRelationships() != null) {
1084                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1085                                                        }
1086                                                        if (temp.getErrors() != null) {
1087                                                                dataflow.getErrors().addAll(temp.getErrors());
1088                                                        }
1089                                                }
1090                                        }
1091                                        else if (sql != null && sqlTrim.startsWith("{")) {
1092                                                EDbVendor vendor = SQLUtil.isEmpty(sqlInfo.getDbVendor()) ? option.getVendor()
1093                                                                : EDbVendor.valueOf(sqlInfo.getDbVendor());
1094                                                TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(),
1095                                                                option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql);
1096                                                if (sqlenvs != null && sqlenvs.length > 0) {
1097                                                        if (sqlenv == null) {
1098                                                                sqlenv = sqlenvs[0];
1099                                                        } else {
1100                                                                sqlenv = SQLEnvParser.mergeSQLEnv(Arrays.asList(sqlenv, sqlenvs[0]));
1101                                                        }
1102                                                }
1103                                                if (sqlenv != null) {
1104                                                        if (MetadataReader.isGrabit(sql) || MetadataReader.isSqlflow(sql)) {
1105                                                                String hash = SHA256.getMd5(sql);
1106                                                                String fileHash = SHA256.getMd5(hash);
1107                                                                if (!sqlInfoMap.containsKey(fileHash)) {
1108                                                                        sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1109                                                                        sqlInfoMap.get(fileHash).add(sqlInfo);
1110                                                                }
1111                                                                ModelBindingManager.setGlobalHash(fileHash);
1112                                                                dataflow temp = null;
1113
1114                                                                if (MetadataReader.isGrabit(sql)) {
1115                                                                        temp = new GrabitMetadataAnalyzer().analyzeMetadata(option.getVendor(), sql);
1116                                                                } else {
1117                                                                        temp = new SqlflowMetadataAnalyzer(sqlenv).analyzeMetadata(option.getVendor(), sql);
1118                                                                }
1119//                                                              if (temp.getPackages() != null) {
1120//                                                                      dataflow.getPackages().addAll(temp.getPackages());
1121//                                                              }
1122//                                                              if (temp.getProcedures() != null) {
1123//                                                                      dataflow.getProcedures().addAll(temp.getProcedures());
1124//                                                              }
1125                                                                if (temp.getTables() != null) {
1126                                                                        dataflow.getTables().addAll(temp.getTables());
1127                                                                }
1128                                                                if (temp.getViews() != null) {
1129                                                                        dataflow.getViews().addAll(temp.getViews());
1130                                                                }
1131                                                                if (temp.getResultsets() != null) {
1132                                                                        dataflow.getResultsets().addAll(temp.getResultsets());
1133                                                                }
1134                                                                if (temp.getRelationships() != null) {
1135                                                                        dataflow.getRelationships().addAll(temp.getRelationships());
1136                                                                }
1137                                                                if (temp.getErrors() != null) {
1138                                                                        dataflow.getErrors().addAll(temp.getErrors());
1139                                                                }
1140                                                                if (sql.indexOf("createdBy") != -1) {
1141                                                                        if (sql.toLowerCase().indexOf("sqldep") != -1
1142                                                                                        || sql.toLowerCase().indexOf("grabit") != -1) {
1143                                                                                Map jsonObject = (Map) JSON.parseObject(sql);
1144                                                                                List<Map> queries = (List<Map>) jsonObject.get("queries");
1145                                                                                if (queries != null) {
1146                                                                                        for (int j = 0; j < queries.size(); j++) {
1147                                                                                                Map queryObject = queries.get(j);
1148                                                                                                appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1149                                                                                        }
1150                                                                                }
1151                                                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
1152                                                                                Map sqlflow = (Map) JSON.parseObject(sql);
1153                                                                                List<Map> servers = (List<Map>) sqlflow.get("servers");
1154                                                                                if (servers != null) {
1155                                                                                        for (Map serverObject : servers) {
1156                                                                                                List<Map> queries = (List<Map>) serverObject.get("queries");
1157                                                                                                if (queries != null) {
1158                                                                                                        for (int j = 0; j < queries.size(); j++) {
1159                                                                                                                Map queryObject = queries.get(j);
1160                                                                                                                appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1161                                                                                                        }
1162                                                                                                }
1163                                                                                        }
1164                                                                                }
1165                                                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
1166                                                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
1167                                                                                        for(Map error: errorMessages){
1168                                                                                                ErrorInfo errorInfo = new ErrorInfo();
1169                                                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
1170                                                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
1171                                                                                                errorInfo.setFileName(sqlInfo.getFileName());
1172                                                                                                errorInfo.setFilePath(sqlInfo.getFilePath());
1173                                                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
1174                                                                                                                ModelBindingManager.getGlobalHash()));
1175                                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
1176                                                                                                                ModelBindingManager.getGlobalHash()));
1177                                                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
1178                                                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
1179                                                                                                metadataErrors.add(errorInfo);
1180                                                                                        }
1181                                                                                }
1182                                                                        }
1183                                                                }
1184                                                        } else {
1185                                                                Map queryObject = (Map) JSON.parseObject(sql);
1186                                                                appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1187                                                        }
1188                                                } else {
1189                                                        Map queryObject = (Map) JSON.parseObject(sql);
1190                                                        appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1191                                                }
1192                                        } else {
1193                                                ModelBindingManager.removeGlobalDatabase();
1194                                                ModelBindingManager.removeGlobalSchema();
1195                                                ModelBindingManager.removeGlobalHash();
1196
1197                                                String content = sql;
1198
1199                                                if (content == null) {
1200                                                        continue;
1201                                                }
1202
1203                                                String delimiterChar = String.valueOf(sqlparser.getDelimiterChar());
1204
1205                                                if (sqlInfos.length > 1) {
1206                                                        String endTrim = SQLUtil.endTrim(content);
1207                                                        if (endTrim.endsWith(delimiterChar) || endTrim.endsWith(";")) {
1208                                                                content += "\n";
1209                                                        } else if (option.getVendor() == EDbVendor.dbvredshift
1210                                                                        || option.getVendor() == EDbVendor.dbvgaussdb
1211                                                                        || option.getVendor() == EDbVendor.dbvedb
1212                                                                        || option.getVendor() == EDbVendor.dbvpostgresql
1213                                                                        || option.getVendor() == EDbVendor.dbvmysql
1214                                                                        || option.getVendor() == EDbVendor.dbvteradata) {
1215                                                                content += ("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
1216                                                        } else {
1217                                                                content = endTrim + ";" + "\n";
1218                                                        }
1219                                                }
1220
1221                                                sqlInfo.setSql(content);
1222
1223                                                if (MetadataReader.isMetadata(content)) {
1224                                                        String hash = SHA256.getMd5(content);
1225                                                        ModelBindingManager.setGlobalHash(hash);
1226                                                        dataflow temp = new SQLDepMetadataAnalyzer().analyzeMetadata(option.getVendor(), content);
1227                                                        if (temp.getProcedures() != null) {
1228                                                                dataflow.getProcedures().addAll(temp.getProcedures());
1229                                                        }
1230                                                        if (temp.getTables() != null) {
1231                                                                dataflow.getTables().addAll(temp.getTables());
1232                                                        }
1233                                                        if (temp.getViews() != null) {
1234                                                                dataflow.getViews().addAll(temp.getViews());
1235                                                        }
1236                                                        if (temp.getResultsets() != null) {
1237                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1238                                                        }
1239                                                        if (temp.getRelationships() != null) {
1240                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1241                                                        }
1242                                                        if (temp.getErrors() != null) {
1243                                                                dataflow.getErrors().addAll(temp.getErrors());
1244                                                        }
1245                                                        String fileHash = SHA256.getMd5(hash);
1246                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1247                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1248                                                                sqlInfoMap.get(fileHash).add(sqlInfo);
1249                                                        }
1250                                                } else {
1251                                                        String sqlHash = SHA256.getMd5(content);
1252                                                        String fileHash = SHA256.getMd5(sqlHash);
1253                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1254                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1255                                                        }
1256
1257                                                        String database = TSQLEnv.DEFAULT_DB_NAME;
1258                                                        String schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1259                                                        if (sqlenv != null) {
1260                                                                database = sqlenv.getDefaultCatalogName();
1261                                                                if (database == null) {
1262                                                                        database = TSQLEnv.DEFAULT_DB_NAME;
1263                                                                }
1264                                                                schema = sqlenv.getDefaultSchemaName();
1265                                                                if (schema == null) {
1266                                                                        schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1267                                                                }
1268                                                        }
1269
1270                                                        boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1271                                                        boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1272                                                        StringBuilder builder = new StringBuilder();
1273                                                        if (supportCatalog) {
1274                                                                builder.append(database);
1275                                                        }
1276                                                        if (supportSchema) {
1277                                                                if (builder.length() > 0) {
1278                                                                        builder.append(".");
1279                                                                }
1280                                                                builder.append(schema);
1281                                                        }
1282                                                        String group = builder.toString();
1283                                                        SqlInfo sqlInfoItem = new SqlInfo();
1284                                                        sqlInfoItem.setFileName(sqlInfo.getFileName());
1285                                                        sqlInfoItem.setFilePath(sqlInfo.getFilePath());
1286                                                        sqlInfoItem.setSql(sqlInfo.getSql());
1287                                                        sqlInfoItem.setOriginIndex(0);
1288                                                        sqlInfoItem.setOriginLineStart(0);
1289                                                        int lineEnd = sqlInfo.getSql().split("\n").length - 1;
1290                                                        sqlInfoItem.setOriginLineEnd(lineEnd);
1291                                                        sqlInfoItem.setIndex(0);
1292                                                        sqlInfoItem.setLineStart(0);
1293                                                        sqlInfoItem.setLineEnd(lineEnd);
1294                                                        sqlInfoItem.setHash(SHA256.getMd5(sqlHash));
1295                                                        sqlInfoItem.setGroup(group);
1296                                                        sqlInfoMap.get(fileHash).add(sqlInfoItem);
1297                                                        if (!databaseMap.containsKey(sqlHash)) {
1298                                                                databaseMap.put(sqlHash, new Pair3<StringBuilder, AtomicInteger, String>(
1299                                                                                new StringBuilder(), new AtomicInteger(), group));
1300                                                        }
1301                                                        databaseMap.get(sqlHash).first.append(sqlInfoItem.getSql());
1302                                                        databaseMap.get(sqlHash).second.incrementAndGet();
1303                                                }
1304                                        }
1305                                }
1306
1307                                boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1308                                boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1309
1310                                Iterator<String> schemaIter = databaseMap.keySet().iterator();
1311                                while (schemaIter.hasNext()) {
1312                                        if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
1313                                                break;
1314                                        }
1315                                        String key = schemaIter.next();
1316                                        String group = databaseMap.get(key).third;
1317                                        String[] split = SQLUtil.parseNames(group).toArray(new String[0]);
1318
1319                                        ModelBindingManager.removeGlobalDatabase();
1320                                        ModelBindingManager.removeGlobalSchema();
1321                                        ModelBindingManager.removeGlobalSQLEnv();
1322                                        ModelBindingManager.removeGlobalHash();
1323
1324                                        String defaultDatabase = null;
1325                                        String defaultSchema = null;
1326                                        if(sqlenv!=null){
1327                                                defaultDatabase = sqlenv.getDefaultCatalogName();
1328                                                defaultSchema = sqlenv.getDefaultSchemaName();
1329                                        }
1330                                        if (supportCatalog && supportSchema) {
1331                                                if (split.length >= 2) {
1332                                                        if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 2])) {
1333                                                                ModelBindingManager.setGlobalDatabase(split[split.length - 2]);
1334                                                                sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1335                                                        }
1336                                                }
1337                                                if (split.length >= 1) {
1338                                                        if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1339                                                                ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1340                                                                sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1341                                                        }
1342                                                }
1343                                        } else if (supportCatalog) {
1344                                                if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 1])) {
1345                                                        ModelBindingManager.setGlobalDatabase(split[split.length - 1]);
1346                                                        sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1347                                                }
1348                                        } else if (supportSchema) {
1349                                                if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1350                                                        ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1351                                                        sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1352                                                }
1353                                        }
1354                                        if (option.getHandleListener() != null) {
1355                                                option.getHandleListener().startParse(null, databaseMap.get(key).first.toString());
1356                                        }
1357
1358                                        if (sqlenv == null) {
1359                                                sqlenv = new TSQLEnv(option.getVendor()) {
1360
1361                                                        @Override
1362                                                        public void initSQLEnv() {
1363                                                                // TODO Auto-generated method stub
1364
1365                                                        }
1366                                                };
1367                                        }
1368                                        ModelBindingManager.setGlobalSQLEnv(sqlenv);
1369                                        sqlparser.sqltext = databaseMap.get(key).first.toString();
1370                                        ModelBindingManager.setGlobalHash(SHA256.getMd5(key));
1371                                        analyzeAndOutputResult(sqlparser);
1372                                        if(sqlenv!=null){
1373                                                sqlenv.setDefaultCatalogName(defaultDatabase);
1374                                                sqlenv.setDefaultSchemaName(defaultSchema);
1375                                        }
1376                                }
1377
1378                                appendProcesses(dataflow);
1379                                appendOraclePackages(dataflow);
1380                                appendProcedures(dataflow);
1381                                appendTables(dataflow);
1382                                appendViews(dataflow);
1383                                appendResultSets(dataflow);
1384                                appendRelations(dataflow);
1385                                appendErrors(dataflow);
1386                        }
1387
1388                        dataflow = handleDataflowExecProcedure(dataflow);
1389
1390                        if (dataflow != null && option.getAnalyzeMode() != AnalyzeMode.crud) {
1391                                if (!isShowJoin()) {
1392                                        dataflow = mergeTables(dataflow, modelManager.TABLE_COLUMN_ID, option);
1393                                } else {
1394                                        dataflow = removeDuplicateColumns(dataflow);
1395                                }
1396                        }
1397
1398                        if (dataflow != null && option.isNormalizeOutput()) {
1399                                dataflow = getNormalizeDataflow(dataflow);
1400                        }
1401
1402                        if (dataflow != null && option.isIgnoreCoordinate()) {
1403                                dataflow = filterDataflowCoordinate(dataflow);
1404                        }
1405                        
1406                        if (option.isSimpleOutput() || option.isIgnoreRecordSet()) {
1407                                List<String> showTypes = new ArrayList<>();
1408                                if(option.getSimpleShowRelationTypes()!=null) {
1409                                        showTypes.addAll(option.getSimpleShowRelationTypes());
1410                                }
1411                                if(showTypes.isEmpty()) {
1412                                        showTypes.add("fdd");
1413                                }
1414                                if(option.isShowCallRelation()) {
1415                                        showTypes.add("call");
1416                                }
1417                                if(option.isShowERDiagram()) {
1418                                        showTypes.add("er");
1419                                }
1420                                dataflow simpleDataflow = getSimpleDataflow(dataflow, option.isSimpleOutput(), showTypes);
1421                                if (simpleDataflow.getResultsets() != null) {
1422                                        for (table t : simpleDataflow.getResultsets()) {
1423                                                t.setIsTarget(null);
1424                                        }
1425                                }
1426                                return simpleDataflow;
1427                        } else {
1428                                return dataflow;
1429                        }
1430                        
1431                } catch (Exception e) {
1432                        logger.error("analyze sql failed.", e);
1433                        ErrorInfo errorInfo = new ErrorInfo();
1434                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
1435                        if (e.getMessage() == null) {
1436                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
1437                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
1438                                } else {
1439                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
1440                                }
1441                        } else {
1442                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
1443                        }
1444                        errorInfo.fillInfo(this);
1445                        errorInfos.add(errorInfo);
1446                }
1447
1448                modelManager.reset();
1449                return null;
1450        }
1451
1452        private dataflow handleDataflowExecProcedure(dataflow dataflow) {
1453                Set<String> procedures = new HashSet<String>();
1454                for (procedure procedure : dataflow.getProcedures()) {
1455                        procedures.add(DlineageUtil.getIdentifierNormalTableName(procedure.getName()));
1456                }
1457                List<table> removeResultSets = new ArrayList<table>();
1458                
1459                Map<String, table> allMap = new HashMap<String, table>();
1460                
1461                if (dataflow.getResultsets() != null) {
1462                        for (table t : dataflow.getResultsets()) {
1463                                allMap.put(t.getId().toLowerCase(), t);
1464                        }
1465                }
1466
1467                if (dataflow.getTables() != null) {
1468                        for (table t : dataflow.getTables()) {
1469                                allMap.put(t.getId().toLowerCase(), t);
1470                        }
1471                }
1472
1473                if (dataflow.getViews() != null) {
1474                        for (table t : dataflow.getViews()) {
1475                                allMap.put(t.getId().toLowerCase(), t);
1476                        }
1477                }
1478
1479                if (dataflow.getVariables() != null) {
1480                        for (table t : dataflow.getVariables()) {
1481                                allMap.put(t.getId().toLowerCase(), t);
1482                        }
1483                }
1484
1485                
1486                for (table resultSet : dataflow.getResultsets()) {
1487                        String resultSetName = DlineageUtil.getIdentifierNormalTableName(resultSet.getName());
1488                        if (!(ResultSetType.function.name().equals(resultSet.getType())
1489                                        && modelManager.getFunctionTable(resultSetName) != null && procedures.contains(resultSetName))) {
1490                                continue;
1491                        }
1492
1493                        Set<Object> functionTables = modelManager
1494                                        .getFunctionTable(DlineageUtil.getIdentifierNormalTableName(resultSet.getName()));
1495                        List<ResultSet> functionResults = functionTables.stream().filter(t -> t instanceof ResultSet)
1496                                        .map(t -> (ResultSet) t).collect(Collectors.toList());
1497                        if (!(resultSet.getColumns().size() == 1
1498                                        && DlineageUtil.getIdentifierNormalTableName(resultSet.getColumns().get(0).getName()).equals(resultSetName))
1499                                        || functionResults.isEmpty()) {
1500                                continue;
1501                        }
1502
1503                        column column = resultSet.getColumns().get(0);
1504                        String sourceColumnId = column.getId();
1505                        boolean reserveResultSet = false;
1506                        List<relationship> appendRelations = new ArrayList<relationship>();
1507                        Set<relationship> removeRelations = new HashSet<relationship>();
1508                        
1509                        for (relationship relationship : dataflow.getRelationships()) {
1510                                targetColumn targetColumn = relationship.getTarget();
1511                                List<sourceColumn> sourceColumns = relationship.getSources();
1512                                List<sourceColumn> newSourceColumns = new ArrayList<sourceColumn>();
1513                                for (sourceColumn sourceColumn : sourceColumns) {
1514                                        if (!sourceColumn.getId().equals(sourceColumnId)) {
1515                                                continue;
1516                                        }
1517                                        for (ResultSet sourceResultSet : functionResults) {
1518                                                for (ResultColumn resultColumn : sourceResultSet.getColumns()) {
1519                                                        if (!DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
1520                                                                        targetColumn.getColumn())) {
1521                                                                if (targetColumn.getColumn().endsWith("*")) {
1522                                                                        table parent = allMap.get(targetColumn.getParent_id());
1523                                                                        if (parent != null && parent.getColumns().size() > 1) {
1524                                                                                for (column item : parent.getColumns()) {
1525                                                                                        if (DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
1526                                                                                                        item.getName())) {
1527                                                                                                relationship newRelation = appendStarRelation(dataflow, relationship, item, resultColumn);
1528                                                                                                appendRelations.add(newRelation);
1529                                                                                                removeRelations.add(relationship);
1530                                                                                                newSourceColumns.add(newRelation.getSources().get(0));
1531                                                                                                if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
1532                                                                                                        dataflow.getResultsets().stream().filter(
1533                                                                                                                        t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
1534                                                                                                                        .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
1535                                                                                                }
1536                                                                                        }
1537                                                                                }
1538                                                                        }
1539                                                                }
1540                                                                continue;
1541                                                        }
1542                                                        
1543                                                        sourceColumn sourceColumn1 = new sourceColumn();
1544                                                        sourceColumn1.setId(String.valueOf(resultColumn.getId()));
1545                                                        sourceColumn1.setColumn(resultColumn.getName());
1546                                                        sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
1547                                                        sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
1548                                                        if (resultColumn.getStartPosition() != null
1549                                                                        && resultColumn.getEndPosition() != null) {
1550                                                                sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
1551                                                                                + "," + convertCoordinate(resultColumn.getEndPosition()));
1552                                                        }
1553                                                        newSourceColumns.add(sourceColumn1);
1554                                                        
1555                                                        if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
1556                                                                dataflow.getResultsets().stream().filter(
1557                                                                                t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
1558                                                                                .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
1559                                                        }
1560                                                }
1561                                        }
1562
1563                                        if (!newSourceColumns.isEmpty()) {
1564                                                if (removeRelations.isEmpty()) {
1565                                                        relationship.setSources(newSourceColumns);
1566                                                }
1567                                        }
1568                                        else {
1569                                                reserveResultSet = true;
1570                                        }
1571                                }
1572                        }
1573                        
1574                        dataflow.getRelationships().addAll(appendRelations);
1575                        dataflow.getRelationships().removeAll(removeRelations);
1576                        
1577                        if(!reserveResultSet) {
1578                                removeResultSets.add(resultSet);
1579                        }
1580                }
1581                dataflow.getResultsets().removeAll(removeResultSets);
1582                return dataflow;
1583        }
1584
1585        private relationship appendStarRelation(dataflow dataflow,
1586                        relationship relationship, column item, ResultColumn resultColumn) {
1587                relationship relationElement = new relationship();
1588                relationElement.setType(relationship.getType());
1589                relationElement.setEffectType(relationship.getEffectType());
1590                relationElement.setSqlHash(relationship.getSqlHash());
1591                relationElement.setSqlComment(relationship.getSqlComment());
1592                relationElement.setProcedureId(relationship.getProcedureId());
1593                relationElement.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
1594                relationElement.setProcessId(relationship.getProcessId());
1595                relationElement.setProcessType(relationship.getProcessType());
1596                
1597                targetColumn targetColumn1 = new targetColumn();
1598                targetColumn1.setId(String.valueOf(item.getId()));
1599                targetColumn1.setColumn(item.getName());
1600                targetColumn1.setParent_id(relationship.getTarget().getParent_id());
1601                targetColumn1.setParent_name(relationship.getTarget().getParent_name());
1602                targetColumn1.setParent_alias(relationship.getTarget().getParent_alias());
1603                if (relationship.getTarget().getCoordinate() != null) {
1604                        targetColumn1.setCoordinate(item.getCoordinate());
1605                }
1606                relationElement.setTarget(targetColumn1);       
1607                
1608                sourceColumn sourceColumn1 = new sourceColumn();
1609                sourceColumn1.setId(String.valueOf(resultColumn.getId()));
1610                sourceColumn1.setColumn(resultColumn.getName());
1611                sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
1612                sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
1613                if (resultColumn.getStartPosition() != null
1614                                && resultColumn.getEndPosition() != null) {
1615                        sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
1616                                        + "," + convertCoordinate(resultColumn.getEndPosition()));
1617                }
1618                relationElement.addSource(sourceColumn1);
1619                return relationElement;
1620        }
1621
1622        private dataflow getNormalizeDataflow(dataflow instance) {
1623                List<table> tables = new ArrayList<>();
1624                if (instance.getResultsets() != null) {
1625                        for (table t : instance.getResultsets()) {
1626                                tables.add(t);
1627                        }
1628                }
1629
1630                if (instance.getTables() != null) {
1631                        for (table t : instance.getTables()) {
1632                                tables.add(t);
1633                        }
1634                }
1635
1636                if (instance.getViews() != null) {
1637                        for (table t : instance.getViews()) {
1638                                tables.add(t);
1639                        }
1640                }
1641
1642                if (instance.getPaths() != null) {
1643                        for (table t : instance.getPaths()) {
1644                                tables.add(t);
1645                        }
1646                }
1647
1648                if (instance.getStages() != null) {
1649                        for (table t : instance.getStages()) {
1650                                tables.add(t);
1651                        }
1652                }
1653
1654                if (instance.getSequences() != null) {
1655                        for (table t : instance.getSequences()) {
1656                                tables.add(t);
1657                        }
1658                }
1659
1660                if (instance.getDatasources() != null) {
1661                        for (table t : instance.getDatasources()) {
1662                                tables.add(t);
1663                        }
1664                }
1665
1666                if (instance.getDatabases() != null) {
1667                        for (table t : instance.getDatabases()) {
1668                                tables.add(t);
1669                        }
1670                }
1671
1672                if (instance.getSchemas() != null) {
1673                        for (table t : instance.getSchemas()) {
1674                                tables.add(t);
1675                        }
1676                }
1677
1678                if (instance.getStreams() != null) {
1679                        for (table t : instance.getStreams()) {
1680                                tables.add(t);
1681                        }
1682                }
1683
1684                if (instance.getVariables() != null) {
1685                        for (table t : instance.getVariables()) {
1686                                tables.add(t);
1687                        }
1688                }
1689
1690                Map<String, String> idNameMap = new HashMap();
1691
1692                for(table table: tables){
1693                        if(table.getDatabase()!=null) {
1694                                table.setDatabase(DlineageUtil.getIdentifierNormalName(table.getDatabase(), ESQLDataObjectType.dotCatalog));
1695                        }
1696                        if(table.getSchema()!=null) {
1697                                table.setSchema(DlineageUtil.getIdentifierNormalName(table.getSchema(), ESQLDataObjectType.dotSchema));
1698                        }
1699                        if(table.getName()!=null) {
1700                                table.setName(DlineageUtil.getIdentifierNormalName(table.getName(), ESQLDataObjectType.dotTable));
1701                                idNameMap.put(table.getId(), table.getName());
1702                        }
1703                        if(table.getColumns()!=null){
1704                                for(column column: table.getColumns()){
1705                                        column.setName(DlineageUtil.getIdentifierNormalName(column.getName(), ESQLDataObjectType.dotColumn));
1706                                        idNameMap.put(column.getId(), column.getName());
1707                                }
1708                        }
1709                }
1710                if(instance.getPackages()!=null){
1711                        for(oraclePackage oraclePackage: instance.getPackages()){
1712                                if(oraclePackage.getDatabase()!=null) {
1713                                        oraclePackage.setDatabase(DlineageUtil.getIdentifierNormalName(oraclePackage.getDatabase(), ESQLDataObjectType.dotCatalog));
1714                                }
1715                                if(oraclePackage.getSchema()!=null) {
1716                                        oraclePackage.setSchema(DlineageUtil.getIdentifierNormalName(oraclePackage.getSchema(), ESQLDataObjectType.dotSchema));
1717                                }
1718                                if(oraclePackage.getName()!=null) {
1719                                        oraclePackage.setName(DlineageUtil.getIdentifierNormalName(oraclePackage.getName(), ESQLDataObjectType.dotTable));
1720                                        idNameMap.put(oraclePackage.getId(), oraclePackage.getName());
1721                                }
1722                                if(oraclePackage.getArguments()!=null){
1723                                        for(argument argument: oraclePackage.getArguments()){
1724                                                argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1725                                                idNameMap.put(argument.getId(), argument.getName());
1726                                        }
1727                                }
1728                                if(oraclePackage.getProcedures()!=null){
1729                                        for(procedure procedure: oraclePackage.getProcedures()){
1730                                                if(procedure.getDatabase()!=null) {
1731                                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
1732                                                }
1733                                                if(procedure.getSchema()!=null) {
1734                                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
1735                                                }
1736                                                if(procedure.getName()!=null) {
1737                                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
1738                                                        idNameMap.put(procedure.getId(), procedure.getName());
1739                                                }
1740                                                for(argument argument: procedure.getArguments()){
1741                                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1742                                                        idNameMap.put(argument.getId(), argument.getName());
1743                                                }
1744                                        }
1745                                }
1746                        }
1747                }
1748                if(instance.getProcedures()!=null){
1749                        for(procedure procedure: instance.getProcedures()){
1750                                if(procedure.getDatabase()!=null) {
1751                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
1752                                }
1753                                if(procedure.getSchema()!=null) {
1754                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
1755                                }
1756                                if(procedure.getName()!=null) {
1757                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
1758                                        idNameMap.put(procedure.getId(), procedure.getName());
1759                                }
1760                                for(argument argument: procedure.getArguments()){
1761                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1762                                        idNameMap.put(argument.getId(), argument.getName());
1763                                }
1764                        }
1765                }
1766                if (instance.getProcesses() != null) {
1767                        for (process process : instance.getProcesses()) {
1768                                if(process.getDatabase()!=null) {
1769                                        process.setDatabase(DlineageUtil.getIdentifierNormalName(process.getDatabase(), ESQLDataObjectType.dotCatalog));
1770                                }
1771                                if(process.getSchema()!=null) {
1772                                        process.setSchema(DlineageUtil.getIdentifierNormalName(process.getSchema(), ESQLDataObjectType.dotSchema));
1773                                }
1774                                if (process.getProcedureId() != null) {
1775                                        process.setProcedureName(DlineageUtil.getIdentifierNormalName(process.getProcedureName(), ESQLDataObjectType.dotTable));
1776                                        idNameMap.put(process.getId(), process.getName());
1777                                }
1778                        }
1779                }
1780
1781                if(instance.getRelationships()!=null){
1782                        for(relationship relation: instance.getRelationships()){
1783                                targetColumn targetColumn = relation.getTarget();
1784                                if(targetColumn != null) {
1785                                        targetColumn.setColumn(idNameMap.get(targetColumn.getId()));
1786                                        targetColumn.setTarget_name(idNameMap.get(targetColumn.getTarget_id()));
1787                                        targetColumn.setParent_name(idNameMap.get(targetColumn.getParent_id()));
1788                                }
1789                                List<sourceColumn> sourceColumns = relation.getSources();
1790                                if(sourceColumns!=null){
1791                                        for(sourceColumn sourceColumn: sourceColumns){
1792                                                sourceColumn.setColumn(idNameMap.get(sourceColumn.getId()));
1793                                                sourceColumn.setSource_name(idNameMap.get(sourceColumn.getSource_id()));
1794                                                sourceColumn.setParent_name(idNameMap.get(sourceColumn.getParent_id()));
1795                                        }
1796                                }
1797                                targetColumn = relation.getCaller();
1798                                if(targetColumn != null) {
1799                                        targetColumn.setName(idNameMap.get(targetColumn.getId()));
1800                                }
1801                                sourceColumns = relation.getCallees();
1802                                if(sourceColumns!=null){
1803                                        for(sourceColumn sourceColumn: sourceColumns){
1804                                                sourceColumn.setName(idNameMap.get(sourceColumn.getId()));
1805                                        }
1806                                }
1807                        }
1808                }
1809
1810
1811                return instance;
1812        }
1813
1814        private dataflow filterDataflowCoordinate(dataflow instance) {
1815                List<table> tables = new ArrayList<>();
1816                if (instance.getResultsets() != null) {
1817                        for (table t : instance.getResultsets()) {
1818                                tables.add(t);
1819                        }
1820                }
1821
1822                if (instance.getTables() != null) {
1823                        for (table t : instance.getTables()) {
1824                                tables.add(t);
1825                        }
1826                }
1827
1828                if (instance.getViews() != null) {
1829                        for (table t : instance.getViews()) {
1830                                tables.add(t);
1831                        }
1832                }
1833
1834                if (instance.getPaths() != null) {
1835                        for (table t : instance.getPaths()) {
1836                                tables.add(t);
1837                        }
1838                }
1839
1840                if (instance.getStages() != null) {
1841                        for (table t : instance.getStages()) {
1842                                tables.add(t);
1843                        }
1844                }
1845
1846                if (instance.getSequences() != null) {
1847                        for (table t : instance.getSequences()) {
1848                                tables.add(t);
1849                        }
1850                }
1851
1852                if (instance.getDatasources() != null) {
1853                        for (table t : instance.getDatasources()) {
1854                                tables.add(t);
1855                        }
1856                }
1857
1858                if (instance.getDatabases() != null) {
1859                        for (table t : instance.getDatabases()) {
1860                                tables.add(t);
1861                        }
1862                }
1863
1864                if (instance.getSchemas() != null) {
1865                        for (table t : instance.getSchemas()) {
1866                                tables.add(t);
1867                        }
1868                }
1869
1870                if (instance.getStreams() != null) {
1871                        for (table t : instance.getStreams()) {
1872                                tables.add(t);
1873                        }
1874                }
1875
1876                if (instance.getVariables() != null) {
1877                        for (table t : instance.getVariables()) {
1878                                tables.add(t);
1879                        }
1880                }
1881
1882                for(table table: tables){
1883                        table.clearCoordinate();
1884                        if(table.getColumns()!=null){
1885                                for(column column: table.getColumns()){
1886                                        column.clearCoordinate();
1887                                }
1888                        }
1889                }
1890                if(instance.getPackages()!=null){
1891                        for(oraclePackage oraclePackage: instance.getPackages()){
1892                                oraclePackage.setCoordinate(null);
1893                                if(oraclePackage.getProcedures()!=null){
1894                                        for(procedure procedure: oraclePackage.getProcedures()){
1895                                                procedure.setCoordinate(null);
1896                                                for(argument argument: procedure.getArguments()){
1897                                                        argument.setCoordinate(null);
1898                                                }
1899                                        }
1900                                }
1901                        }
1902                }
1903                if(instance.getProcedures()!=null){
1904                        for(procedure procedure: instance.getProcedures()){
1905                                procedure.setCoordinate(null);
1906                                for(argument argument: procedure.getArguments()){
1907                                        argument.setCoordinate(null);
1908                                }
1909                        }
1910                }
1911                if (instance.getProcesses() != null) {
1912                        for (process process : instance.getProcesses()) {
1913                                process.setCoordinate(null);
1914                        }
1915                }
1916
1917                if(instance.getRelationships()!=null){
1918                        for(relationship relation: instance.getRelationships()){
1919                                targetColumn targetColumn = relation.getTarget();
1920                                if(targetColumn != null) {
1921                                        targetColumn.setCoordinate(null);
1922                                }
1923                                List<sourceColumn> sourceColumns = relation.getSources();
1924                                if(sourceColumns!=null){
1925                                        for(sourceColumn sourceColumn: sourceColumns){
1926                                                sourceColumn.setCoordinate(null);
1927                                        }
1928                                }
1929                                targetColumn = relation.getCaller();
1930                                if(targetColumn != null) {
1931                                        targetColumn.setCoordinate(null);
1932                                }
1933                                sourceColumns = relation.getCallees();
1934                                if(sourceColumns!=null){
1935                                        for(sourceColumn sourceColumn: sourceColumns){
1936                                                sourceColumn.setCoordinate(null);
1937                                        }
1938                                }
1939                        }
1940                }
1941
1942                return instance;
1943        }
1944
1945        private void appendSqlInfo(Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, int index,
1946                        SqlInfo sqlInfo, Map queryObject) {
1947                EDbVendor vendor = option.getVendor();
1948                if (!SQLUtil.isEmpty(sqlInfo.getDbVendor())) {
1949                        vendor = EDbVendor.valueOf(sqlInfo.getDbVendor());
1950                }
1951
1952                boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
1953                boolean supportSchema = TSQLEnv.supportSchema(vendor);
1954
1955                String content = (String) queryObject.get("sourceCode");
1956                if (SQLUtil.isEmpty(content)) {
1957                        return;
1958                }
1959                StringBuilder builder = new StringBuilder();
1960                if (supportCatalog) {
1961                        String database = (String) queryObject.get("database");
1962                        if (database.indexOf(".") != -1) {
1963                                String delimitedChar = TSQLEnv.delimitedChar(vendor);
1964                                database = delimitedChar + SQLUtil.trimColumnStringQuote(database) + delimitedChar;
1965                        }
1966                        builder.append(database);
1967                }
1968                if (supportSchema) {
1969                        String schema = (String) queryObject.get("schema");
1970                        if (schema.indexOf(".") != -1) {
1971                                String delimitedChar = TSQLEnv.delimitedChar(vendor);
1972                                schema = delimitedChar + SQLUtil.trimColumnStringQuote(schema) + delimitedChar;
1973                        }
1974                        if (builder.length() > 0) {
1975                                builder.append(".");
1976                        }
1977                        builder.append(schema);
1978                }
1979                String group = builder.toString();
1980                String sqlHash = SHA256.getMd5(content);
1981                String hash = SHA256.getMd5(sqlHash);
1982                if (!databaseMap.containsKey(sqlHash)) {
1983                        databaseMap.put(sqlHash,
1984                                        new Pair3<StringBuilder, AtomicInteger, String>(new StringBuilder(), new AtomicInteger(), group));
1985                }
1986                TGSqlParser parser = new TGSqlParser(option.getVendor());
1987                String delimiterChar = String.valueOf(parser.getDelimiterChar());
1988                StringBuilder buffer = new StringBuilder(content);
1989                if (content.trim().endsWith(delimiterChar) || content.trim().endsWith(";")) {
1990                        buffer.append("\n");
1991                } else if(vendor == EDbVendor.dbvredshift
1992                                || vendor == EDbVendor.dbvgaussdb
1993                                || vendor == EDbVendor.dbvedb
1994                                || vendor == EDbVendor.dbvpostgresql
1995                                || vendor == EDbVendor.dbvmysql
1996                                || vendor == EDbVendor.dbvteradata){
1997                        buffer.append("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
1998                } else{
1999                        SQLUtil.endTrim(buffer);
2000                        buffer.append(";").append("\n");
2001                }
2002
2003                int lineStart = databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1;
2004                if (databaseMap.get(sqlHash).first.toString().length() == 0) {
2005                        lineStart = 0;
2006                }
2007                databaseMap.get(sqlHash).first.append(buffer.toString());
2008                SqlInfo sqlInfoItem = new SqlInfo();
2009                sqlInfoItem.setServer(sqlInfo.getServer());
2010                sqlInfoItem.setDbVendor(sqlInfo.getDbVendor());
2011                sqlInfoItem.setFileName(sqlInfo.getFileName());
2012                sqlInfoItem.setFilePath(sqlInfo.getFilePath());
2013                sqlInfoItem.setSql(buffer.toString());
2014                sqlInfoItem.setOriginIndex(index);
2015                sqlInfoItem.setOriginLineStart(0);
2016                sqlInfoItem.setOriginLineEnd(buffer.toString().split("\n", -1).length - 1);
2017                sqlInfoItem.setIndex(databaseMap.get(sqlHash).second.getAndIncrement());
2018                sqlInfoItem.setLineStart(lineStart);
2019                sqlInfoItem.setLineEnd(databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1);
2020                sqlInfoItem.setGroup(group);
2021                sqlInfoItem.setHash(hash);
2022
2023                if (!sqlInfoMap.containsKey(hash)) {
2024                        sqlInfoMap.put(hash, new ArrayList<SqlInfo>());
2025                }
2026                sqlInfoMap.get(hash).add(sqlInfoItem);
2027        }
2028
2029        static String getTextOutput(dataflow dataflow) {
2030                StringBuffer buffer = new StringBuffer();
2031                List<relationship> relations = dataflow.getRelationships();
2032                if (relations != null) {
2033                        for (int i = 0; i < relations.size(); i++) {
2034                                relationship relation = relations.get(i);
2035                                targetColumn target = relation.getTarget();
2036                                List<sourceColumn> sources = relation.getSources();
2037                                if (target != null && sources != null && sources.size() > 0) {
2038                                        buffer.append(target.getColumn()).append(" depends on: ");
2039                                        Set<String> columnSet = new LinkedHashSet<String>();
2040                                        for (int j = 0; j < sources.size(); j++) {
2041                                                sourceColumn sourceColumn = sources.get(j);
2042                                                String columnName = sourceColumn.getColumn();
2043                                                if (sourceColumn.getParent_name() != null && sourceColumn.getParent_name().length() > 0) {
2044                                                        columnName = sourceColumn.getParent_name() + "." + columnName;
2045                                                }
2046                                                columnSet.add(columnName);
2047                                        }
2048                                        String[] columns = columnSet.toArray(new String[0]);
2049                                        for (int j = 0; j < columns.length; j++) {
2050                                                buffer.append(columns[j]);
2051                                                if (j == columns.length - 1) {
2052                                                        buffer.append("\n");
2053                                                } else
2054                                                        buffer.append(", ");
2055                                        }
2056                                }
2057                        }
2058                }
2059                return buffer.toString();
2060        }
2061
2062        private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) {
2063                RelationshipType relationType = RelationshipType.join;
2064                for (int i = 0; i < typePaths.size(); i++) {
2065                        List<String> path = typePaths.get(i).second;
2066                        RelationshipType type = RelationshipType.valueOf(getRelationType(path));
2067                        if (type.ordinal() < relationType.ordinal()) {
2068                                relationType = type;
2069                        }
2070                }
2071                return relationType.name();
2072        }
2073
2074        private String getRelationType(List<String> typePaths) {
2075                if (typePaths.contains("join"))
2076                        return "join";
2077                if (typePaths.contains("fdr"))
2078                        return "fdr";
2079                if (typePaths.contains("frd"))
2080                        return "frd";
2081                if (typePaths.contains("fddi"))
2082                        return "fddi";
2083                return "fdd";
2084        }
2085
2086        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput) throws Exception {
2087                return getSimpleDataflow(instance, simpleOutput, Arrays.asList("fdd"));
2088        }
2089        
2090        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput, List<String> types) throws Exception {
2091                ModelBindingManager.setGlobalVendor(option.getVendor());
2092                allMap.clear();
2093                targetTables.clear();
2094                resultSetMap.clear();
2095                tableMap.clear();
2096                viewMap.clear();
2097                cursorMap.clear();
2098                variableMap.clear();
2099                fileMap.clear();
2100                stageMap.clear();
2101                sequenceMap.clear();
2102                dataSourceMap.clear();
2103                databaseMap.clear();
2104                schemaMap.clear();
2105                streamMap.clear();
2106                dataflow simple = new dataflow();
2107                List<relationship> simpleRelations = new ArrayList<relationship>();
2108                List<relationship> relations = instance.getRelationships();
2109                if (instance.getResultsets() != null) {
2110                        for (table t : instance.getResultsets()) {
2111                                resultSetMap.put(t.getId().toLowerCase(), t);
2112                                allMap.put(t.getId().toLowerCase(), t);
2113                        }
2114                }
2115
2116                if (instance.getTables() != null) {
2117                        for (table t : instance.getTables()) {
2118                                tableMap.put(t.getId().toLowerCase(), t);
2119                                allMap.put(t.getId().toLowerCase(), t);
2120                        }
2121                }
2122
2123                if (instance.getViews() != null) {
2124                        for (table t : instance.getViews()) {
2125                                viewMap.put(t.getId().toLowerCase(), t);
2126                                allMap.put(t.getId().toLowerCase(), t);
2127                        }
2128                }
2129
2130                if (instance.getPaths() != null) {
2131                        for (table t : instance.getPaths()) {
2132                                fileMap.put(t.getId().toLowerCase(), t);
2133                                allMap.put(t.getId().toLowerCase(), t);
2134                        }
2135                }
2136
2137                if (instance.getStages() != null) {
2138                        for (table t : instance.getStages()) {
2139                                stageMap.put(t.getId().toLowerCase(), t);
2140                                allMap.put(t.getId().toLowerCase(), t);
2141                        }
2142                }
2143                
2144                if (instance.getSequences() != null) {
2145                        for (table t : instance.getSequences()) {
2146                                sequenceMap.put(t.getId().toLowerCase(), t);
2147                                allMap.put(t.getId().toLowerCase(), t);
2148                        }
2149                }
2150
2151                if (instance.getDatasources() != null) {
2152                        for (table t : instance.getDatasources()) {
2153                                dataSourceMap.put(t.getId().toLowerCase(), t);
2154                                allMap.put(t.getId().toLowerCase(), t);
2155                        }
2156                }
2157
2158                if (instance.getDatabases() != null) {
2159                        for (table t : instance.getDatabases()) {
2160                                databaseMap.put(t.getId().toLowerCase(), t);
2161                                allMap.put(t.getId().toLowerCase(), t);
2162                        }
2163                }
2164
2165                if (instance.getSchemas() != null) {
2166                        for (table t : instance.getSchemas()) {
2167                                schemaMap.put(t.getId().toLowerCase(), t);
2168                                allMap.put(t.getId().toLowerCase(), t);
2169                        }
2170                }
2171
2172                if (instance.getStreams() != null) {
2173                        for (table t : instance.getStreams()) {
2174                                streamMap.put(t.getId().toLowerCase(), t);
2175                                allMap.put(t.getId().toLowerCase(), t);
2176                        }
2177                }
2178
2179                if (instance.getVariables() != null) {
2180                        for (table t : instance.getVariables()) {
2181                                if(SubType.cursor.name().equals(t.getSubType())){
2182                                        cursorMap.put(t.getId().toLowerCase(), t);
2183                                }
2184                                else {
2185                                        variableMap.put(t.getId().toLowerCase(), t);
2186                                }
2187                                allMap.put(t.getId().toLowerCase(), t);
2188                        }
2189                }
2190
2191                if (relations != null) {
2192                        
2193                        List<relationship> filterRelations = new ArrayList<>();
2194                        for (relationship relationElem : relations) {
2195                                if (!types.contains(relationElem.getType()))
2196                                        continue;
2197                                else {
2198                                        filterRelations.add(relationElem);
2199                                }
2200                        }
2201                        
2202                        relations = filterRelations;
2203                        
2204                        Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
2205                        for (relationship relation : relations) {
2206                                if (relation.getTarget() != null) {
2207                                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
2208                                        if (!targetIdRelationMap.containsKey(key)) {
2209                                                targetIdRelationMap.put(key, new TreeSet<relationship>(new Comparator<relationship>() {
2210                                                        @Override
2211                                                        public int compare(relationship o1, relationship o2) {
2212                                                                return o1.getId().compareTo(o2.getId());
2213                                                        }
2214                                                }));
2215                                        }
2216                                        targetIdRelationMap.get(key).add(relation);
2217                                }
2218                        }
2219
2220                        Iterator<String> keys = targetIdRelationMap.keySet().iterator();
2221                        while (keys.hasNext()) {
2222                                String key = keys.next();
2223                                if (targetIdRelationMap.get(key).size() > 500) {
2224                                        keys.remove();
2225                                }
2226                        }
2227
2228                        for (relationship relationElem : relations) {
2229                                if (RelationshipType.call.name().equals(relationElem.getType())) {
2230                                        continue;
2231                                }
2232                                if (RelationshipType.er.name().equals(relationElem.getType())) {
2233                                        continue;
2234                                }
2235                                targetColumn target = relationElem.getTarget();
2236                                String targetParent = target.getParent_id();
2237                                if (isTarget(instance, targetParent, simpleOutput)) {
2238                                        List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>();
2239                                        findSourceRelations(target, instance, targetIdRelationMap, relationElem, relationSources,
2240                                                        new String[] { relationElem.getType() }, simpleOutput);
2241                                        if (relationSources.size() > 0) {
2242                                                Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new LinkedHashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>();
2243                                                for (Pair<sourceColumn, List<String>> t : relationSources) {
2244                                                        sourceColumn key = ((Pair<sourceColumn, List<String>>) t).first;
2245                                                        if (!columnMap.containsKey(key)) {
2246                                                                columnMap.put(key, new ArrayList<Pair<sourceColumn, List<String>>>());
2247                                                        }
2248                                                        columnMap.get(key).add(t);
2249                                                }
2250                                                Iterator<sourceColumn> iter = columnMap.keySet().iterator();
2251                                                Map<String, List<sourceColumn>> relationSourceMap = new HashMap<String, List<sourceColumn>>();
2252                                                while (iter.hasNext()) {
2253                                                        sourceColumn column = iter.next();
2254                                                        String relationType = mergeRelationType(columnMap.get(column));
2255                                                        if (!relationSourceMap.containsKey(relationType)) {
2256                                                                relationSourceMap.put(relationType, new ArrayList<sourceColumn>());
2257                                                        }
2258                                                        relationSourceMap.get(relationType).add(column);
2259                                                }
2260
2261                                                Iterator<String> sourceIter = relationSourceMap.keySet().iterator();
2262                                                while (sourceIter.hasNext()) {
2263                                                        String relationType = sourceIter.next();
2264                                                        relationship simpleRelation = (relationship) relationElem.clone();
2265                                                        simpleRelation.setSources(relationSourceMap.get(relationType));
2266                                                        simpleRelation.setType(relationType);
2267                                                        simpleRelation.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
2268                                                        simpleRelations.add(simpleRelation);
2269                                                }
2270                                        }
2271                                }
2272                        }
2273                }
2274
2275                simple.setProcedures(instance.getProcedures());
2276                simple.setPackages(instance.getPackages());
2277                simple.setProcesses(instance.getProcesses());
2278                simple.setErrors(instance.getErrors());
2279                List<table> tables = new ArrayList<table>();
2280                for (table t : instance.getTables()) {
2281                        if (!SQLUtil.isTempTable(t)) {
2282                                tables.add(t);
2283                        }
2284                        else {
2285                                if (option.isIgnoreTemporaryTable()) {
2286                                        continue;
2287                                }
2288                                else {
2289                                        tables.add(t);
2290                                }
2291                        }
2292                }
2293                simple.setStages(instance.getStages());
2294                simple.setSequences(instance.getSequences());
2295                simple.setDatasources(instance.getDatasources());
2296                simple.setStreams(instance.getStreams());
2297                simple.setPaths(instance.getPaths());
2298                simple.setTables(tables);
2299                simple.setViews(instance.getViews());
2300                if(option.isSimpleShowVariable()) {
2301                        simple.setVariables(instance.getVariables());
2302                }
2303                else if(option.isSimpleShowCursor()) {
2304                        simple.setVariables(instance.getVariables().stream().filter(t->SubType.cursor.name().equals(t.getSubType())).collect(Collectors.toList()));
2305                }
2306                if (instance.getResultsets() != null) {
2307                        List<table> resultSets = new ArrayList<table>();
2308                        for (int i = 0; i < instance.getResultsets().size(); i++) {
2309                                table resultSet = instance.getResultsets().get(i);
2310                                if (isTargetResultSet(instance, resultSet.getId(), simpleOutput)) {
2311                                        // special handle function #524 #296
2312                                        resultSets.add(resultSet);
2313                                }
2314                        }
2315                        simple.setResultsets(resultSets);
2316                }
2317
2318                List<table> functions = new ArrayList<table>();
2319                if (option.isShowCallRelation()) {
2320                        for (int i = 0; i < relations.size(); i++) {
2321                                relationship relationElem = relations.get(i);
2322                                if (!RelationshipType.call.name().equals(relationElem.getType())) {
2323                                        continue;
2324                                }
2325                                simpleRelations.add(relationElem);
2326                                for (sourceColumn callee : relationElem.getCallees()) {
2327                                        String calleeId = callee.getId();
2328                                        if (resultSetMap.containsKey(calleeId)) {
2329                                                table function = resultSetMap.get(calleeId);
2330                                                function.setIsTarget("true");
2331                                                functions.add(function);
2332                                        }
2333                                }
2334                        }
2335                }
2336
2337                if (option.isShowERDiagram()) {
2338                        for (int i = 0; i < relations.size(); i++) {
2339                                relationship relationElem = relations.get(i);
2340                                if (!RelationshipType.er.name().equals(relationElem.getType())) {
2341                                        continue;
2342                                }
2343                                simpleRelations.add(relationElem);
2344                                for (sourceColumn callee : relationElem.getCallees()) {
2345                                        String calleeId = callee.getId();
2346                                        if (resultSetMap.containsKey(calleeId)) {
2347                                                table function = resultSetMap.get(calleeId);
2348                                                function.setIsTarget("true");
2349                                                functions.add(function);
2350                                        }
2351                                }
2352                        }
2353                }
2354                
2355                if (!functions.isEmpty()) {
2356                        if (simple.getResultsets() == null) {
2357                                simple.setResultsets(functions);
2358                        } else {
2359                                simple.getResultsets().addAll(functions);
2360                        }
2361                }
2362
2363                simple.setRelationships(simpleRelations);
2364                simple.setOrientation(instance.getOrientation());
2365                targetTables.clear();
2366                resultSetMap.clear();
2367                tableMap.clear();
2368                viewMap.clear();
2369                cursorMap.clear();
2370                variableMap.clear();
2371                fileMap.clear();
2372                stageMap.clear();
2373                dataSourceMap.clear();
2374                databaseMap.clear();
2375                schemaMap.clear();
2376                streamMap.clear();
2377                return simple;
2378        }
2379
2380        private void findSourceRelations(targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap,
2381                        relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, boolean simpleOutput) {
2382                findStarSourceRelations(target, instance, null, sourceIdRelationMap, targetRelation, relationSources, pathTypes,
2383                                new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>(), 0, simpleOutput);
2384        }
2385
2386        private void findStarSourceRelations(targetColumn target, dataflow instance, targetColumn starRelationTarget,
2387                        Map<String, Set<relationship>> sourceIdRelationMap, relationship targetRelation,
2388                        List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, Set<String> paths,
2389                        Set<transform> transforms, Set<candidateTable> candidateTables, int level, boolean simpleOutput) {
2390                if (targetRelation != null && targetRelation.getSources() != null) {
2391                        
2392                        //获取source为*的Column Parent
2393                        String starParentId = null;
2394                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
2395                                sourceColumn source = targetRelation.getSources().get(i);
2396                                if (starRelationTarget != null && "*".equals(source.getColumn())) {
2397                                        starParentId = source.getParent_id();
2398                                }
2399                        }
2400                        
2401                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
2402                                sourceColumn source = targetRelation.getSources().get(i);
2403                                if (starRelationTarget != null && !"*".equals(source.getColumn())
2404                                                && !DlineageUtil.getIdentifierNormalColumnName(starRelationTarget.getColumn())
2405                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(source.getColumn()))) {
2406                                        table parent = allMap.get(source.getParent_id());
2407                                        if (parent != null && isFunction(parent)) {
2408                                                // function返回值未知，不对星号做处理
2409                                        } 
2410                                        else if (parent == null) {
2411                                                continue;
2412                                        } else if(starParentId!=null && starParentId.equals(parent.getId())){
2413                                                //如果source和 * column的parent相同，则跳过
2414                                                continue;
2415                                        }
2416                                }
2417
2418                                String sourceColumnId = source.getId();
2419                                String sourceParentId = source.getParent_id();
2420                                if (sourceParentId == null || sourceColumnId == null) {
2421                                        continue;
2422                                }
2423                                if (isTarget(instance, sourceParentId, simpleOutput)) {
2424                                        List<transform> transforms2 = new ArrayList<transform>(transforms.size());
2425                                        transforms2.addAll(transforms);
2426                                        Collections.reverse(transforms2);
2427                                        
2428                                        List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size());
2429                                        candidateTables2.addAll(candidateTables);
2430                                        
2431                                        sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source);
2432                                        for (transform t : transforms2) {
2433                                                sourceColumnCopy.addTransform(t);
2434                                        }
2435                                        
2436                                        for (candidateTable t : candidateTables2) {
2437                                                sourceColumnCopy.addCandidateParent(t);
2438                                        }
2439                                        
2440                                        if(Boolean.TRUE.equals(target.isStruct()) && Boolean.TRUE.equals(source.isStruct())) {
2441                                                List<String> targetColumns = SQLUtil.parseNames(target.getColumn());
2442                                                List<String> sourceColumns = SQLUtil.parseNames(source.getColumn());
2443                                                if(!DlineageUtil.getIdentifierNormalColumnName(targetColumns.get(targetColumns.size()-1))
2444                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(sourceColumns.get(sourceColumns.size()-1)))) {
2445                                                        continue;
2446                                                }
2447                                        }
2448                                        relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes)));
2449                                } else {
2450                                        Set<relationship> sourceRelations = sourceIdRelationMap
2451                                                        .get(source.getParent_id() + "." + source.getId());
2452                                        if (sourceRelations != null) {
2453                                                if (paths.contains(source.getParent_id() + "." + source.getId())) {
2454                                                        continue;
2455                                                } else {
2456                                                        paths.add(source.getParent_id() + "." + source.getId());
2457                                                        if (source.getTransforms() != null) {
2458                                                                transforms.addAll(source.getTransforms());
2459                                                        }
2460                                                        if (source.getCandidateParents() != null) {
2461                                                                candidateTables.addAll(source.getCandidateParents());
2462                                                        }
2463                                                }
2464                                                for (relationship relation : sourceRelations) {
2465                                                        LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size());
2466                                                        transforms2.addAll(transforms);
2467                                                        LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size());
2468                                                        candidateTables2.addAll(candidateTables);
2469                                                        String[] types = new String[pathTypes.length + 1];
2470                                                        types[0] = relation.getType();
2471                                                        System.arraycopy(pathTypes, 0, types, 1, pathTypes.length);
2472                                                        if (!"*".equals(source.getColumn())) {
2473                                                                findStarSourceRelations(target, instance, null, sourceIdRelationMap, relation, relationSources,
2474                                                                                types, paths, transforms2, candidateTables2, level + 1, simpleOutput);
2475                                                        } else {
2476                                                                findStarSourceRelations(target, instance,
2477                                                                                starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget,
2478                                                                                sourceIdRelationMap, relation, relationSources, types, paths, transforms, candidateTables2,
2479                                                                                level + 1, simpleOutput);
2480                                                        }
2481                                                }
2482                                        }
2483                                }
2484                        }
2485                }
2486        }
2487
2488        private Map<String, Boolean> targetTables = new HashMap<String, Boolean>();
2489        private Map<String, table> resultSetMap = new HashMap<String, table>();
2490        private Map<String, table> tableMap = new HashMap<String, table>();
2491        private Map<String, table> viewMap = new HashMap<String, table>();
2492        private Map<String, table> cursorMap = new HashMap<String, table>();
2493        private Map<String, table> variableMap = new HashMap<String, table>();
2494        private Map<String, table> fileMap = new HashMap<String, table>();
2495        private Map<String, table> stageMap = new HashMap<String, table>();
2496        private Map<String, table> sequenceMap = new HashMap<String, table>();
2497        private Map<String, table> dataSourceMap = new HashMap<String, table>();
2498        private Map<String, table> databaseMap = new HashMap<String, table>();
2499        private Map<String, table> schemaMap = new HashMap<String, table>();
2500        private Map<String, table> streamMap = new HashMap<String, table>();
2501        private Map<String, table> allMap = new HashMap<String, table>();
2502
2503        private boolean isTarget(dataflow instance, String targetParentId, boolean simpleOutput) {
2504                if (targetTables.containsKey(targetParentId))
2505                        return targetTables.get(targetParentId);
2506                if (isTable(instance, targetParentId)) {
2507                        targetTables.put(targetParentId, true);
2508                        return true;
2509                } else if (isView(instance, targetParentId)) {
2510                        targetTables.put(targetParentId, true);
2511                        return true;
2512                } else if (isFile(instance, targetParentId)) {
2513                        targetTables.put(targetParentId, true);
2514                        return true;
2515                } else if (isDatabase(instance, targetParentId)) {
2516                        targetTables.put(targetParentId, true);
2517                        return true;
2518                } else if (isSchema(instance, targetParentId)) {
2519                        targetTables.put(targetParentId, true);
2520                        return true;
2521                } else if (isStage(instance, targetParentId)) {
2522                        targetTables.put(targetParentId, true);
2523                        return true;
2524                } else if (isSequence(instance, targetParentId)) {
2525                        targetTables.put(targetParentId, true);
2526                        return true;
2527                } else if (isDataSource(instance, targetParentId)) {
2528                        targetTables.put(targetParentId, true);
2529                        return true;
2530                } else if (isStream(instance, targetParentId)) {
2531                        targetTables.put(targetParentId, true);
2532                        return true;
2533                } else if (isCursor(instance, targetParentId) && option.isSimpleShowCursor()) {
2534                        targetTables.put(targetParentId, true);
2535                        return true;
2536                } else if ((isVariable(instance, targetParentId) || isCursor(instance, targetParentId)) && option.isSimpleShowVariable()) {
2537                        targetTables.put(targetParentId, true);
2538                        return true;
2539                } else if (isTargetResultSet(instance, targetParentId, simpleOutput)) {
2540                        targetTables.put(targetParentId, true);
2541                        return true;
2542                }
2543                targetTables.put(targetParentId, false);
2544                return false;
2545        }
2546
2547        private boolean isTargetResultSet(dataflow instance, String targetParent, boolean simpleOutput) {
2548                if (resultSetMap.containsKey(targetParent.toLowerCase())) {
2549                        table result = resultSetMap.get(targetParent.toLowerCase());
2550                        boolean isTarget = result.isTarget();
2551                        Option option = ModelBindingManager.getGlobalOption();
2552                        if (option != null && option.isSqlflowIgnoreFunction() && isFunction(result)) {
2553                                return false;
2554                        }
2555                        if (isTarget && simpleOutput) {
2556                                if (option != null && option.isSimpleShowFunction() && isFunction(result)) {
2557                                        return true;
2558
2559                                } else if (option != null && option.isSimpleShowTopSelectResultSet()) {
2560                                        return true;
2561                                }
2562                                if (ResultSetType.of(result.getType()) != null && option.containsResultSetType(ResultSetType.of(result.getType()))) {
2563                                        return true;
2564                                }
2565                        } else
2566                                return isTarget;
2567                }
2568                return false;
2569        }
2570
2571        private boolean isFunction(table resultSet) {
2572                if("function".equals(resultSet.getType())){
2573                        return true;
2574                }
2575                else if("resultset".equals(resultSet.getType()) && "function".equals(resultSet.getSubType())){
2576                        return true;
2577                }
2578                return false;
2579        }
2580
2581        private boolean isView(dataflow instance, String targetParent) {
2582                if (viewMap.containsKey(targetParent.toLowerCase())) {
2583                        return true;
2584                }
2585                return false;
2586        }
2587
2588        private boolean isCursor(dataflow instance, String targetParent) {
2589                if (cursorMap.containsKey(targetParent.toLowerCase())) {
2590                        return true;
2591                }
2592                return false;
2593        }
2594
2595        private boolean isVariable(dataflow instance, String targetParent) {
2596                if (variableMap.containsKey(targetParent.toLowerCase())) {
2597                        return true;
2598                }
2599                return false;
2600        }
2601
2602        private boolean isFile(dataflow instance, String targetParent) {
2603                if (fileMap.containsKey(targetParent.toLowerCase())) {
2604                        return true;
2605                }
2606                return false;
2607        }
2608
2609        private boolean isStage(dataflow instance, String targetParent) {
2610                if (stageMap.containsKey(targetParent.toLowerCase())) {
2611                        return true;
2612                }
2613                return false;
2614        }
2615        
2616        private boolean isSequence(dataflow instance, String targetParent) {
2617                if (sequenceMap.containsKey(targetParent.toLowerCase())) {
2618                        return true;
2619                }
2620                return false;
2621        }
2622
2623        private boolean isDataSource(dataflow instance, String targetParent) {
2624                if (dataSourceMap.containsKey(targetParent.toLowerCase())) {
2625                        return true;
2626                }
2627                return false;
2628        }
2629
2630        private boolean isDatabase(dataflow instance, String targetParent) {
2631                if (databaseMap.containsKey(targetParent.toLowerCase())) {
2632                        return true;
2633                }
2634                return false;
2635        }
2636
2637        private boolean isSchema(dataflow instance, String targetParent) {
2638                if (schemaMap.containsKey(targetParent.toLowerCase())) {
2639                        return true;
2640                }
2641                return false;
2642        }
2643
2644        private boolean isStream(dataflow instance, String targetParent) {
2645                if (streamMap.containsKey(targetParent.toLowerCase())) {
2646                        return true;
2647                }
2648                return false;
2649        }
2650
2651        private boolean isTable(dataflow instance, String targetParent) {
2652                if (tableMap.containsKey(targetParent.toLowerCase())) {
2653                        if (SQLUtil.isTempTable(tableMap.get(targetParent))) {
2654                                if (option.isIgnoreTemporaryTable()) {
2655                                        return false;
2656                                }
2657                        }
2658                        if (tableMap.get(targetParent).isFunction()) {
2659                                if (option != null && option.isSimpleShowFunction()) {
2660                                        return true;
2661                                } else {
2662                                        return false;
2663                                }
2664                        }
2665                        if (SubType.synonym.name().equals(tableMap.get(targetParent).getSubType())) {
2666                                if (option != null && option.isSimpleShowSynonym()) {
2667                                        return true;
2668                                } else {
2669                                        return false;
2670                                }
2671                        }
2672                        return true;
2673                } else {
2674                        return false;
2675                }
2676        }
2677
2678        private void init() {
2679                metadataErrors.clear();
2680                sqlInfoMap.clear();
2681                errorInfos.clear();
2682                dataflow = null;
2683                dataflowString = null;
2684                ModelBindingManager.removeGlobalDatabase();
2685                ModelBindingManager.removeGlobalSchema();
2686                ModelBindingManager.removeGlobalVendor();
2687                ModelBindingManager.removeGlobalSQLEnv();
2688                ModelBindingManager.removeGlobalHash();
2689                appendResultSets.clear();
2690                appendStarColumns.clear();
2691                appendTableStarColumns.clear();
2692                modelManager.TABLE_COLUMN_ID = option.getStartId();
2693                modelManager.RELATION_ID = option.getStartId();
2694                modelManager.DISPLAY_ID.clear();
2695                modelManager.DISPLAY_NAME.clear();
2696                tableIds.clear();
2697                ModelBindingManager.setGlobalVendor(option.getVendor());
2698                modelManager.reset();
2699        }
2700
2701        private String getErrorMessage(TSyntaxError error, String errorType) {
2702                String s = "", hint = "Syntax error";
2703                if (ErrorInfo.SYNTAX_HINT.equals(errorType)) {
2704                        hint = "Syntax hint";
2705                }
2706                if (error.hint.length() > 0)
2707                        hint = error.hint;
2708                s = s + hint + "(" + error.errorno + ") near: " + error.tokentext;
2709                s = s + "(" + error.lineNo;
2710                s = s + "," + error.columnNo + ")";
2711                return s;
2712        }
2713
2714//      boolean OLD_ENABLE_RESOLVER = TBaseType.isEnableResolver();
2715        private void analyzeAndOutputResult(TGSqlParser sqlparser) {
2716                try {
2717                        accessedSubqueries.clear();
2718                        accessedStatements.clear();
2719                        stmtStack.clear();
2720                        viewDDLMap.clear();
2721                        procedureDDLMap.clear();
2722                        structObjectMap.clear();
2723                        try {
2724                                if(sqlenv!=null) {
2725                                        sqlparser.setSqlEnv(sqlenv);
2726                                }
2727                                int result = sqlparser.parse();
2728                                if (result != 0) {
2729                                        ArrayList<TSyntaxError> errors = sqlparser.getSyntaxErrors();
2730                                        if (errors != null && !errors.isEmpty()) {
2731                                                for (int i = 0; i < errors.size(); i++) {
2732                                                        TSyntaxError error = errors.get(i);
2733                                                        ErrorInfo errorInfo = new ErrorInfo();
2734                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
2735                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
2736                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
2737                                                                        ModelBindingManager.getGlobalHash()));
2738                                                        String[] segments = error.tokentext.split("\n");
2739                                                        if (segments.length <= 1) {
2740                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
2741                                                                                error.columnNo + error.tokentext.length(),
2742                                                                                ModelBindingManager.getGlobalHash()));
2743                                                        } else {
2744                                                                errorInfo.setEndPosition(
2745                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
2746                                                                                                (long) segments[segments.length - 1].length() + 1,
2747                                                                                                ModelBindingManager.getGlobalHash()));
2748                                                        }
2749                                                        ;
2750                                                        errorInfo.fillInfo(this);
2751                                                        errorInfos.add(errorInfo);
2752                                                }
2753                                        }
2754                                }
2755
2756                                if (option.getHandleListener() != null) {
2757                                        option.getHandleListener().endParse(result == 0);
2758                                }
2759                        } catch (Exception e) {
2760                                logger.error("analyze sql failed.", e);
2761                                if (option.getHandleListener() != null) {
2762                                        option.getHandleListener().endParse(false);
2763                                }
2764                                ErrorInfo errorInfo = new ErrorInfo();
2765                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
2766                                if (e.getMessage() == null) {
2767                                        if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
2768                                                errorInfo
2769                                                                .setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
2770                                        } else {
2771                                                errorInfo.setErrorMessage(e.getClass().getSimpleName());
2772                                        }
2773                                } else {
2774                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
2775                                }
2776                                errorInfo.fillInfo(this);
2777                                errorInfos.add(errorInfo);
2778                                return;
2779                        }
2780
2781                        
2782                        TSQLResolver2 resolver = sqlparser.getResolver2();
2783                        if (resolver != null && option.getVendor() == EDbVendor.dbvbigquery) {
2784                                ScopeBuildResult buildResult = resolver.getScopeBuildResult();
2785                                List<TObjectName> columns = buildResult.getAllColumnReferences();
2786                                for (TObjectName col : columns) {
2787                                        structObjectMap.putIfAbsent(col.getSourceTable(), new TObjectNameList());
2788                                        structObjectMap.get(col.getSourceTable()).addObjectName(col);
2789                                }
2790                        }
2791                        
2792                        if (option.getHandleListener() != null) {
2793                                option.getHandleListener().startAnalyzeDataFlow(sqlparser);
2794                        }
2795
2796                        for (int i = 0; i < sqlparser.getSqlstatements().size(); i++) {
2797                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2798                                        break;
2799                                }
2800
2801                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2802                                if (stmt.getErrorCount() == 0) {
2803                                        if (stmt.getParentStmt() == null) {
2804                                                modelManager.collectSqlHash(stmt);
2805                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
2806                                                                || stmt instanceof TCreateTableSqlStatement
2807                                                                || stmt instanceof TCreateExternalDataSourceStmt || stmt instanceof TCreateStageStmt
2808                                                                || stmt instanceof TMssqlCreateType
2809                                                                || stmt instanceof TMssqlDeclare
2810                                                                || stmt instanceof TPlsqlCreateType_Placeholder
2811                                                                || stmt instanceof TPlsqlCreateType
2812                                                                || stmt instanceof TPlsqlTableTypeDefStmt
2813                                                                || (stmt instanceof TCreateFunctionStmt && hasDb2ReturnStmt((TCreateFunctionStmt)stmt))
2814                                                                || (stmt instanceof TMssqlCreateFunction
2815                                                                                && (((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null
2816                                                                                                || ((TMssqlCreateFunction) stmt).getReturnStmt() != null))) {
2817                                                        boolean listen = false;
2818                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2819                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2820                                                                listen = true;
2821                                                        }
2822                                                        analyzeCustomSqlStmt(stmt);
2823                                                        if (listen && option.getHandleListener() != null) {
2824                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2825                                                        }
2826                                                }
2827                                        }
2828                                }
2829                        }
2830
2831                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2832                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2833                                        break;
2834                                }
2835
2836                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2837                                if (stmt.getErrorCount() == 0) {
2838                                        if (stmt.getParentStmt() == null) {
2839                                                if (stmt instanceof TUseDatabase 
2840                                                                || stmt instanceof TUseSchema
2841                                                                || stmt instanceof TCreateViewSqlStatement 
2842                                                                || stmt instanceof TCreateSynonymStmt 
2843                                                                || stmt instanceof TStoredProcedureSqlStatement) {
2844                                                        boolean listen = false;
2845                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2846                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2847                                                                listen = true;
2848                                                        }
2849                                                        if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema) {
2850                                                                analyzeCustomSqlStmt(stmt);
2851                                                        } else if (stmt instanceof TCreateViewSqlStatement) {
2852                                                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
2853                                                                if(view.getViewName()!=null) {
2854                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(view.getViewName().toString()), view);
2855                                                                }
2856                                                        } else if (stmt instanceof TCreateSynonymStmt) {
2857                                                                TCreateSynonymStmt synonym = (TCreateSynonymStmt) stmt;
2858                                                                if(synonym.getSynonymName()!=null) {
2859                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(synonym.getSynonymName().toString()), synonym);
2860                                                                }
2861                                                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
2862                                                                TStoredProcedureSqlStatement procedure = (TStoredProcedureSqlStatement) stmt;
2863                                                                if(procedure.getStoredProcedureName() == null) {
2864                                                                        continue;
2865                                                                }
2866                                                                procedureDDLMap.put(DlineageUtil.getProcedureNameWithArgNum(procedure), procedure);
2867                                                        }
2868                                                        if (listen && option.getHandleListener() != null) {
2869                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2870                                                        }
2871                                                }
2872                                        }
2873                                }
2874                        }
2875
2876                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2877                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2878                                        break;
2879                                }
2880
2881                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2882                                if (stmt.getErrorCount() == 0) {
2883                                        if (stmt.getParentStmt() == null) {
2884                                                if (stmt instanceof TUseDatabase 
2885                                                                || stmt instanceof TUseSchema
2886                                                                || stmt instanceof TCreateViewSqlStatement 
2887                                                                || stmt instanceof TCreateSynonymStmt) {
2888                                                        boolean listen = false;
2889                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2890                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2891                                                                listen = true;
2892                                                        }
2893                                                        analyzeCustomSqlStmt(stmt);
2894                                                        if (listen && option.getHandleListener() != null) {
2895                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2896                                                        }
2897                                                }
2898                                        }
2899                                }
2900                        }
2901
2902                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2903                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2904                                        break;
2905                                }
2906
2907                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2908                                if (stmt.getErrorCount() == 0) {
2909                                        if (stmt.getParentStmt() == null) {
2910                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
2911                                                                || stmt instanceof TStoredProcedureSqlStatement) {
2912                                                        boolean listen = false;
2913                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2914                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2915                                                                listen = true;
2916                                                        }
2917                                                        if (stmt instanceof TPlsqlCreateTrigger)
2918                                                                continue;
2919                                                        analyzeCustomSqlStmt(stmt);
2920                                                        if (listen && option.getHandleListener() != null) {
2921                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2922                                                        }
2923                                                }
2924                                        }
2925                                }
2926                        }
2927
2928                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2929                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2930                                        break;
2931                                }
2932
2933                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2934
2935                                if (option.isIgnoreTopSelect()) {
2936                                        if ((stmt instanceof TSelectSqlStatement && ((TSelectSqlStatement)stmt).getIntoClause() == null && ((TSelectSqlStatement)stmt).getIntoTableClause() == null ) || stmt instanceof TRedshiftDeclare
2937                                                        || stmt instanceof TRedshiftDeclare) {
2938                                                continue;
2939                                        }
2940                                }
2941
2942                                if (stmt.getErrorCount() == 0) {
2943                                        if (stmt.getParentStmt() == null) {
2944                                                if (!(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TCreateStageStmt)
2945                                                                && !(stmt instanceof TCreateExternalDataSourceStmt)
2946                                                                && !(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TMssqlDeclare)
2947                                                                && !(stmt instanceof TMssqlCreateFunction
2948                                                                                && ((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null)) {
2949                                                        boolean listen = false;
2950                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2951                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2952                                                                listen = true;
2953                                                        }
2954                                                        analyzeCustomSqlStmt(stmt);
2955                                                        if (listen && option.getHandleListener() != null) {
2956                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2957                                                        }
2958                                                }
2959                                        }
2960                                }
2961                        }
2962
2963                        if (option.getHandleListener() != null) {
2964                                option.getHandleListener().endAnalyzeDataFlow(sqlparser);
2965                        }
2966
2967                        // Finalize pipelined function stitching after all passes
2968                        if (!modelManager.getPendingPipelinedCallSites().isEmpty() && sqlparser.getSqlstatements().size() > 0 && pipelinedAnalyzer != null) {
2969                                TCustomSqlStatement lastStmt = sqlparser.getSqlstatements().get(sqlparser.getSqlstatements().size() - 1);
2970                                stmtStack.push(lastStmt);
2971                                try {
2972                                        pipelinedAnalyzer.stitchPendingCallSites();
2973                                } catch (Exception ex) {
2974                                        // Don't let pipelined stitching failure break main flow
2975                                } finally {
2976                                        stmtStack.pop();
2977                                }
2978                        }
2979                } catch (Throwable e) {
2980                        logger.error("analyze sql failed.", e);
2981                        ErrorInfo errorInfo = new ErrorInfo();
2982                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
2983                        if (e.getMessage() == null) {
2984                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
2985                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
2986                                } else {
2987                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
2988                                }
2989                        } else {
2990                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
2991                        }
2992                        errorInfo.fillInfo(this);
2993                        errorInfos.add(errorInfo);
2994                }
2995
2996        }
2997
2998        private boolean hasDb2ReturnStmt(TCreateFunctionStmt stmt) {
2999                if (stmt.getReturnStmt() != null) {
3000                        return true;
3001                }
3002                if (stmt.getBodyStatements() != null) {
3003                        for (int i = 0; i < stmt.getBodyStatements().size(); i++) {
3004                                if(stmt.getBodyStatements().get(i) instanceof TDb2ReturnStmt) {
3005                                        return true;
3006                                }
3007                        }
3008                }
3009                return false;
3010        }
3011
3012        private void analyzeCustomSqlStmt(TCustomSqlStatement stmt) {
3013                if (!accessedStatements.contains(stmt)) {
3014                        accessedStatements.add(stmt);
3015                } else if (!(stmt instanceof TUseDatabase || stmt instanceof TUseSchema)) {
3016                        return;
3017                }
3018
3019                ArrayList<TSyntaxError> errors = stmt.getSyntaxHints();
3020                if (errors != null && !errors.isEmpty()) {
3021                        for (int i = 0; i < errors.size(); i++) {
3022                                TSyntaxError error = errors.get(i);
3023                                ErrorInfo errorInfo = new ErrorInfo();
3024                                errorInfo.setErrorType(ErrorInfo.SYNTAX_HINT);
3025                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_HINT));
3026                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3027                                                ModelBindingManager.getGlobalHash()));
3028                                String[] segments = error.tokentext.split("\n");
3029                                if (segments.length == 1) {
3030                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3031                                                        error.columnNo + error.tokentext.length(), ModelBindingManager.getGlobalHash()));
3032                                } else {
3033                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3034                                                        (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
3035                                }
3036                                errorInfo.fillInfo(this);
3037                                errorInfos.add(errorInfo);
3038                        }
3039                }
3040
3041                if (option.getAnalyzeMode() == AnalyzeMode.dynamic) {
3042                        if (!(stmt instanceof TStoredProcedureSqlStatement
3043                                        || stmt instanceof TExecuteSqlStatement
3044                                        || stmt instanceof TMssqlExecute
3045                                        || stmt instanceof TExecImmeStmt)) {
3046                                return;
3047                        }
3048                }
3049
3050                if(DlineageUtil.getTopStmt(stmt) == stmt){
3051                        modelManager.collectSqlHash(stmt);
3052                }
3053
3054                try {
3055                        if (stmt instanceof TUseDatabase) {
3056                                if (((TUseDatabase) stmt).getDatabaseName() != null) {
3057                                        ModelBindingManager.setGlobalDatabase(((TUseDatabase) stmt).getDatabaseName().toString());
3058                                }
3059                        } else if (stmt instanceof TUseSchema) {
3060                                if (((TUseSchema) stmt).getSchemaName() != null) {
3061                                        String schemaName = ((TUseSchema) stmt).getSchemaName().toString();
3062                                        List<String> splits = SQLUtil.parseNames(schemaName);
3063                                        if (splits.size() == 1) {
3064                                                ModelBindingManager.setGlobalSchema(schemaName);
3065                                        } else if (splits.size() > 1) {
3066                                                ModelBindingManager.setGlobalSchema(splits.get(splits.size() - 1));
3067                                                ModelBindingManager.setGlobalDatabase(splits.get(splits.size() - 2));
3068                                        }
3069                                }
3070                        } else if (stmt instanceof TPlsqlRecordTypeDefStmt) {
3071                                this.stmtStack.push(stmt);
3072                                this.analyzePlsqlRecordTypeDefStmt((TPlsqlRecordTypeDefStmt) stmt);
3073                                this.stmtStack.pop();
3074                        } else if (stmt instanceof TPlsqlCreateType_Placeholder) {
3075                                this.stmtStack.push(stmt);
3076                                TPlsqlCreateType_Placeholder placeholder = (TPlsqlCreateType_Placeholder) stmt;
3077                                if (placeholder.getObjectStatement() != null && pipelinedAnalyzer != null) {
3078                                        this.pipelinedAnalyzer.indexObjectType(placeholder.getObjectStatement());
3079                                }
3080                                if (placeholder.getNestedTableStatement() != null) {
3081                                        if (pipelinedAnalyzer != null) {
3082                                                this.pipelinedAnalyzer.indexCollectionType(placeholder.getNestedTableStatement());
3083                                        }
3084                                        this.analyzePlsqlTableTypeDefStmt(placeholder.getNestedTableStatement());
3085                                }
3086                                this.stmtStack.pop();
3087                        } else if (stmt instanceof TPlsqlCreateType) {
3088                                this.stmtStack.push(stmt);
3089                                if (pipelinedAnalyzer != null) {
3090                                        this.pipelinedAnalyzer.indexObjectType((TPlsqlCreateType) stmt);
3091                                }
3092                                this.stmtStack.pop();
3093                        } else if (stmt instanceof TPlsqlTableTypeDefStmt) {
3094                                this.stmtStack.push(stmt);
3095                                this.analyzePlsqlTableTypeDefStmt((TPlsqlTableTypeDefStmt) stmt);
3096                                if (pipelinedAnalyzer != null) {
3097                                        this.pipelinedAnalyzer.indexCollectionType((TPlsqlTableTypeDefStmt) stmt);
3098                                }
3099                                this.stmtStack.pop();
3100                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
3101                                this.stmtStack.push(stmt);
3102                                this.analyzeStoredProcedureStmt((TStoredProcedureSqlStatement) stmt);
3103                                this.stmtStack.pop();
3104                        } else if (stmt instanceof TCreateTableSqlStatement) {
3105                                stmtStack.push(stmt);
3106                                analyzeCreateTableStmt((TCreateTableSqlStatement) stmt);
3107                                stmtStack.pop();
3108                        } else if (stmt instanceof TCreateStageStmt) {
3109                                stmtStack.push(stmt);
3110                                analyzeCreateStageStmt((TCreateStageStmt) stmt);
3111                                stmtStack.pop();
3112                        } else if (stmt instanceof TCreateExternalDataSourceStmt) {
3113                                stmtStack.push(stmt);
3114                                analyzeCreateExternalDataSourceStmt((TCreateExternalDataSourceStmt) stmt);
3115                                stmtStack.pop();
3116                        } else if (stmt instanceof TCreateStreamStmt) {
3117                                stmtStack.push(stmt);
3118                                analyzeCreateStreamStmt((TCreateStreamStmt) stmt);
3119                                stmtStack.pop();
3120                        } else if (stmt instanceof TSelectSqlStatement) {
3121                                analyzeSelectStmt((TSelectSqlStatement) stmt);
3122                        } else if (stmt instanceof TDropTableSqlStatement) {
3123                                stmtStack.push(stmt);
3124                                analyzeDropTableStmt((TDropTableSqlStatement) stmt);
3125                                stmtStack.pop();
3126                        } else if (stmt instanceof TTruncateStatement) {
3127                                stmtStack.push(stmt);
3128                                analyzeTruncateTableStmt((TTruncateStatement) stmt);
3129                                stmtStack.pop();
3130                        } else if (stmt instanceof TCreateMaterializedSqlStatement) {
3131                                stmtStack.push(stmt);
3132                                TCreateMaterializedSqlStatement view = (TCreateMaterializedSqlStatement) stmt;
3133                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
3134                                stmtStack.pop();
3135                        } else if (stmt instanceof TCreateViewSqlStatement) {
3136                                stmtStack.push(stmt);
3137                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
3138                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
3139                                stmtStack.pop();
3140                        } else if(stmt instanceof TDb2SqlVariableDeclaration){
3141                                stmtStack.push(stmt);
3142                                analyzeDb2Declare((TDb2SqlVariableDeclaration)stmt);
3143                                stmtStack.pop();
3144                        } else if (stmt instanceof TMssqlCreateType) {
3145                                stmtStack.push(stmt);
3146                                TMssqlCreateType createType = (TMssqlCreateType) stmt;
3147                                analyzeMssqlCreateType(createType);
3148                                stmtStack.pop();
3149                        } else if (stmt instanceof TMssqlDeclare) {
3150                                stmtStack.push(stmt);
3151                                TMssqlDeclare declare = (TMssqlDeclare) stmt;
3152                                analyzeMssqlDeclare(declare);
3153                                stmtStack.pop();
3154                        } else if (stmt instanceof TInsertSqlStatement) {
3155                                stmtStack.push(stmt);
3156                                TInsertSqlStatement insert = (TInsertSqlStatement)stmt;
3157                                analyzeInsertStmt(insert);
3158                                if(insert.getMultiInsertStatements()!=null) {
3159                                        for(int i=0;i<insert.getMultiInsertStatements().size();i++) {
3160                                                analyzeInsertStmt(insert.getMultiInsertStatements().get(i));
3161                                        }
3162                                }
3163                                stmtStack.pop();
3164                        } else if (stmt instanceof TRedshiftCopy) {
3165                                stmtStack.push(stmt);
3166                                analyzeRedshiftCopyStmt((TRedshiftCopy) stmt);
3167                                stmtStack.pop();
3168                        } else if (stmt instanceof TSnowflakeCopyIntoStmt) {
3169                                stmtStack.push(stmt);
3170                                analyzeCopyIntoStmt((TSnowflakeCopyIntoStmt) stmt);
3171                                stmtStack.pop();
3172                        } else if (stmt instanceof TUnloadStmt) {
3173                                stmtStack.push(stmt);
3174                                analyzeUnloadStmt((TUnloadStmt) stmt);
3175                                stmtStack.pop();
3176                        } else if (stmt instanceof TUpdateSqlStatement) {
3177                                stmtStack.push(stmt);
3178                                analyzeUpdateStmt((TUpdateSqlStatement) stmt);
3179                                stmtStack.pop();
3180                        } else if (stmt instanceof TMergeSqlStatement) {
3181                                stmtStack.push(stmt);
3182                                analyzeMergeStmt((TMergeSqlStatement) stmt);
3183                                stmtStack.pop();
3184                        } else if (stmt instanceof TDeleteSqlStatement) {
3185                                stmtStack.push(stmt);
3186                                analyzeDeleteStmt((TDeleteSqlStatement) stmt);
3187                                stmtStack.pop();
3188                        } else if (stmt instanceof TCursorDeclStmt) {
3189                                stmtStack.push(stmt);
3190                                analyzeCursorDeclStmt((TCursorDeclStmt) stmt);
3191                                stmtStack.pop();
3192                        } else if (stmt instanceof TFetchStmt) {
3193                                stmtStack.push(stmt);
3194                                analyzeFetchStmt((TFetchStmt) stmt);
3195                                stmtStack.pop();
3196                        } else if (stmt instanceof TMssqlFetch) {
3197                                stmtStack.push(stmt);
3198                                analyzeFetchStmt((TMssqlFetch) stmt);
3199                                stmtStack.pop();
3200                        } else if (stmt instanceof TForStmt) {
3201                                stmtStack.push(stmt);
3202                                analyzeForStmt((TForStmt) stmt);
3203                                stmtStack.pop();
3204                        } else if (stmt instanceof TOpenforStmt) {
3205                                stmtStack.push(stmt);
3206                                analyzeOpenForStmt((TOpenforStmt) stmt);
3207                                stmtStack.pop();
3208                        } else if (stmt instanceof TLoopStmt) {
3209                                stmtStack.push(stmt);
3210                                analyzeLoopStmt((TLoopStmt) stmt);
3211                                stmtStack.pop();
3212                        } else if (stmt instanceof TAssignStmt) {
3213                                stmtStack.push(stmt);
3214                                analyzeAssignStmt((TAssignStmt) stmt);
3215                                stmtStack.pop();
3216                        } else if (stmt instanceof TSetStmt) {
3217                                stmtStack.push(stmt);
3218                                analyzeSetStmt((TSetStmt) stmt);
3219                                stmtStack.pop();
3220                        } else if (stmt instanceof TMssqlSet) {
3221                                stmtStack.push(stmt);
3222                                analyzeMssqlSetStmt((TMssqlSet) stmt);
3223                                stmtStack.pop();
3224                        } else if (stmt instanceof TVarDeclStmt) {
3225                                stmtStack.push(stmt);
3226                                analyzeVarDeclStmt((TVarDeclStmt) stmt);
3227                                stmtStack.pop();
3228                        } else if (stmt instanceof TCreateDatabaseSqlStatement) {
3229                                stmtStack.push(stmt);
3230                                analyzeCloneDatabaseStmt((TCreateDatabaseSqlStatement) stmt);
3231                                stmtStack.pop();
3232                        } else if (stmt instanceof TCreateSchemaSqlStatement) {
3233                                stmtStack.push(stmt);
3234                                analyzeCloneSchemaStmt((TCreateSchemaSqlStatement) stmt);
3235                                stmtStack.pop();
3236                        } else if (stmt instanceof TAlterTableStatement) {
3237                                stmtStack.push(stmt);
3238                                analyzeAlterTableStmt((TAlterTableStatement) stmt);
3239                                stmtStack.pop();
3240                        } else if (stmt instanceof TAlterViewStatement) {
3241                                stmtStack.push(stmt);
3242                                analyzeAlterViewStmt((TAlterViewStatement) stmt);
3243                                stmtStack.pop();
3244                        } else if (stmt instanceof TRenameStmt) {
3245                                stmtStack.push(stmt);
3246                                analyzeRenameStmt((TRenameStmt) stmt);
3247                                stmtStack.pop();
3248                        } else if (stmt instanceof TCreateSynonymStmt) {
3249                                stmtStack.push(stmt);
3250                                analyzeCreateSynonymStmt((TCreateSynonymStmt) stmt);
3251                                stmtStack.pop();
3252                        } else if (stmt instanceof TLoadDataStmt) {
3253                                stmtStack.push(stmt);
3254                                analyzeLoadDataStmt((TLoadDataStmt) stmt);
3255                                stmtStack.pop();
3256                        } else if (stmt instanceof THiveLoad) {
3257                                stmtStack.push(stmt);
3258                                analyzeHiveLoadStmt((THiveLoad) stmt);
3259                                stmtStack.pop();
3260                        } else if (stmt instanceof TDb2ReturnStmt) {
3261                                stmtStack.push(stmt);
3262                                analyzeDb2ReturnStmt((TDb2ReturnStmt) stmt);
3263                                stmtStack.pop();
3264                        } else if (stmt instanceof TReturnStmt) {
3265                                stmtStack.push(stmt);
3266                                analyzeReturnStmt((TReturnStmt) stmt);
3267                                stmtStack.pop();
3268                        } else if (stmt instanceof TMssqlReturn) {
3269                                stmtStack.push(stmt);
3270                                analyzeMssqlReturnStmt((TMssqlReturn) stmt);
3271                                stmtStack.pop();
3272                        } else if (stmt instanceof TExecuteSqlStatement) {
3273                                String sqlText = ((TExecuteSqlStatement) stmt).getPreparedSqlText();
3274                                if(sqlText == null) {
3275                                        sqlText = ((TExecuteSqlStatement) stmt).getSqlText();
3276                                }
3277                                if (sqlText != null) {
3278                                        modelManager.collectDynamicSqlHash(stmt);
3279                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3280                                        sqlparser.sqltext = SQLUtil.trimColumnStringQuote(sqlText);
3281                                        int result = sqlparser.parse();
3282                                        if (result != 0) {
3283                                                errors = sqlparser.getSyntaxErrors();
3284                                                if (errors != null && !errors.isEmpty()) {
3285                                                        for (int i = 0; i < errors.size(); i++) {
3286                                                                TSyntaxError error = errors.get(i);
3287                                                                ErrorInfo errorInfo = new ErrorInfo();
3288                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3289                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3290                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3291                                                                                ModelBindingManager.getGlobalHash()));
3292                                                                String[] segments = error.tokentext.split("\n");
3293                                                                if (segments.length == 1) {
3294                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3295                                                                                        error.columnNo + error.tokentext.length(),
3296                                                                                        ModelBindingManager.getGlobalHash()));
3297                                                                } else {
3298                                                                        errorInfo.setEndPosition(
3299                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3300                                                                                                        (long) segments[segments.length - 1].length() + 1,
3301                                                                                                        ModelBindingManager.getGlobalHash()));
3302                                                                }
3303                                                                errorInfo.fillInfo(this);
3304                                                                errorInfos.add(errorInfo);
3305                                                        }
3306                                                }
3307                                        } else if (sqlparser.sqlstatements != null) {
3308                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3309                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3310                                                }
3311                                        }
3312                                }
3313                                else if (((TExecuteSqlStatement) stmt).getStmtString() != null) {
3314                                        modelManager.collectDynamicSqlHash(stmt);
3315                                }
3316                        } else if (stmt instanceof TMssqlExecute) {
3317                                TMssqlExecute executeStmt = (TMssqlExecute)stmt;
3318                                if (executeStmt.getSqlText() != null) {
3319                                        modelManager.collectDynamicSqlHash(stmt);
3320                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3321                                        sqlparser.sqltext = ((TMssqlExecute) stmt).getSqlText();
3322                                        int result = sqlparser.parse();
3323                                        if (result != 0) {
3324                                                errors = sqlparser.getSyntaxErrors();
3325                                                if (errors != null && !errors.isEmpty()) {
3326                                                        for (int i = 0; i < errors.size(); i++) {
3327                                                                TSyntaxError error = errors.get(i);
3328                                                                ErrorInfo errorInfo = new ErrorInfo();
3329                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3330                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3331                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3332                                                                                ModelBindingManager.getGlobalHash()));
3333                                                                String[] segments = error.tokentext.split("\n");
3334                                                                if (segments.length == 1) {
3335                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3336                                                                                        error.columnNo + error.tokentext.length(),
3337                                                                                        ModelBindingManager.getGlobalHash()));
3338                                                                } else {
3339                                                                        errorInfo.setEndPosition(
3340                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3341                                                                                                        (long) segments[segments.length - 1].length() + 1,
3342                                                                                                        ModelBindingManager.getGlobalHash()));
3343                                                                }
3344                                                                errorInfo.fillInfo(this);
3345                                                                errorInfos.add(errorInfo);
3346                                                        }
3347                                                }
3348                                        } else if (sqlparser.sqlstatements != null) {
3349                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3350                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3351                                                }
3352                                        }
3353                                } else if (executeStmt.getModuleName() != null) {
3354                                        stmtStack.push(stmt);
3355                                        analyzeMssqlExecute(executeStmt);
3356                                        stmtStack.pop();
3357                                }
3358                        } else if (stmt instanceof TExecImmeStmt) {
3359                                TExecImmeStmt execImmeStmt = (TExecImmeStmt) stmt;
3360                                modelManager.collectDynamicSqlHash(stmt);
3361                                synchronized (DataFlowAnalyzer.class) {
3362                                        TStatementList stmts = execImmeStmt.getDynamicStatements();
3363                                        if (stmts != null && stmts.size() > 0) {
3364                                                for (int i = 0; i < stmts.size(); i++) {
3365                                                        analyzeCustomSqlStmt(stmts.get(i));
3366                                                }
3367                                        }
3368
3369                                        String dynamicSql = execImmeStmt.getDynamicSQL();
3370                                        if (!SQLUtil.isEmpty(dynamicSql)) {
3371                                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3372                                                sqlparser.sqltext = dynamicSql;
3373                                                int result = sqlparser.parse();
3374                                                if (result != 0) {
3375                                                        errors = sqlparser.getSyntaxErrors();
3376                                                        if (errors != null && !errors.isEmpty()) {
3377                                                                for (int i = 0; i < errors.size(); i++) {
3378                                                                        TSyntaxError error = errors.get(i);
3379                                                                        ErrorInfo errorInfo = new ErrorInfo();
3380                                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3381                                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3382                                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3383                                                                                        ModelBindingManager.getGlobalHash()));
3384                                                                        String[] segments = error.tokentext.split("\n");
3385                                                                        if (segments.length == 1) {
3386                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3387                                                                                                error.columnNo + error.tokentext.length(),
3388                                                                                                ModelBindingManager.getGlobalHash()));
3389                                                                        } else {
3390                                                                                errorInfo.setEndPosition(
3391                                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3392                                                                                                                (long) segments[segments.length - 1].length() + 1,
3393                                                                                                                ModelBindingManager.getGlobalHash()));
3394                                                                        }
3395                                                                        errorInfo.fillInfo(this);
3396                                                                        errorInfos.add(errorInfo);
3397                                                                }
3398                                                        }
3399                                                } else if (sqlparser.sqlstatements != null) {
3400                                                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3401                                                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3402                                                        }
3403                                                }
3404                                        }
3405                                }
3406                        } else if (stmt instanceof TCallStatement) {
3407                                TCallStatement callStmt = (TCallStatement)stmt;
3408                                stmtStack.push(stmt);
3409                                analyzeCallStmt(callStmt);
3410                                stmtStack.pop();
3411                        } else if (stmt instanceof TBasicStmt) {
3412                                TBasicStmt oracleBasicStmt = (TBasicStmt) stmt;
3413                                stmtStack.push(stmt);
3414                                analyzeOracleBasicStmt(oracleBasicStmt);
3415                                stmtStack.pop();
3416                        } else if (stmt instanceof TIfStmt) {
3417                                TIfStmt ifStmt = (TIfStmt) stmt;
3418                                stmtStack.push(stmt);
3419                                analyzeIfStmt(ifStmt);
3420                                stmtStack.pop();
3421                        } else if (stmt instanceof TElsifStmt) {
3422                                TElsifStmt elsIfStmt = (TElsifStmt) stmt;
3423                                stmtStack.push(stmt);
3424                                analyzeElsIfStmt(elsIfStmt);
3425                                stmtStack.pop();
3426                        } else if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
3427                                for (int i = 0; i < stmt.getStatements().size(); i++) {
3428                                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
3429                                }
3430                        } else if (stmt instanceof TCreateIndexSqlStatement) {
3431                                stmtStack.push(stmt);
3432                                analyzeCreateIndexStageStmt((TCreateIndexSqlStatement) stmt);
3433                                stmtStack.pop();
3434                        } else if (stmt instanceof gudusoft.gsqlparser.stmt.mdx.TMdxSelect) {
3435                                stmtStack.push(stmt);
3436                                analyzeMdxSelectStmt((gudusoft.gsqlparser.stmt.mdx.TMdxSelect) stmt);
3437                                stmtStack.pop();
3438                        }
3439                } catch (Exception e) {
3440            StringBuffer errorMessage = new StringBuffer();
3441            errorMessage.append("analyze sql stmt failed, ");
3442            if (stmt.getStartToken() != null) {
3443                errorMessage.append("line: " + stmt.getStartToken().lineNo + ", column: " + stmt.getStartToken().columnNo).append(", ");
3444            }
3445            if (stmt.getGsqlparser() != null && !SQLUtil.isEmpty(stmt.getGsqlparser().sqlfilename)) {
3446                errorMessage.append("file: "+ stmt.getGsqlparser().sqlfilename).append(", ");
3447            }
3448            if (stmt.toString() != null) {
3449                errorMessage.append("sql:\n" + stmt.toString());
3450            }
3451            logger.error(errorMessage.toString(), e);
3452                        ErrorInfo errorInfo = new ErrorInfo();
3453                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
3454                        if (e.getMessage() == null) {
3455                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
3456                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
3457                                } else {
3458                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
3459                                }
3460                        } else {
3461                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
3462                        }
3463                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
3464                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
3465                        String[] segments = stmt.getEndToken().getAstext().split("\n", -1);
3466                        if (segments.length == 1) {
3467                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
3468                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
3469                                                ModelBindingManager.getGlobalHash()));
3470                        } else {
3471                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1,
3472                                                (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
3473                        }
3474                        errorInfo.fillInfo(this);
3475                        errorInfos.add(errorInfo);
3476                }
3477        }
3478
3479        private void analyzePlsqlRecordTypeDefStmt(TPlsqlRecordTypeDefStmt stmt) {
3480                TObjectName typeName = stmt.getTypeName();
3481                Variable variable = modelFactory.createVariable(typeName);
3482                variable.setSubType(SubType.record_type);
3483        
3484                if (stmt.getFieldDeclarations() != null) {
3485                        for (int i = 0; i < stmt.getFieldDeclarations().size(); i++) {
3486                                TParameterDeclaration param = stmt.getFieldDeclarations().getParameterDeclarationItem(i);
3487                                String dataTypeName = param.getDataType().getDataTypeName();
3488                                TObjectName columnName = param.getParameterName();      
3489                                TableColumn variableProperty = modelFactory.createTableColumn(variable, columnName, true);
3490                                if(dataTypeName.indexOf(".")!=-1) {
3491                                        String tableName = dataTypeName.substring(0, dataTypeName.lastIndexOf("."));
3492                                        Table table = modelFactory.createTableByName(tableName, true);
3493                                        if(table!=null) {
3494                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(table, dataTypeName);
3495                                                if (tableColumn != null) {
3496                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
3497                                                        relation.setEffectType(EffectType.rowtype);
3498                                                        relation.setTarget(new TableColumnRelationshipElement(variableProperty));
3499                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
3500                                                }
3501                                        }
3502                                }
3503                        }
3504                        variable.setDetermined(true);
3505                }
3506                else {
3507                        TObjectName starColumn = new TObjectName();
3508                        starColumn.setString("*");
3509                }
3510        }
3511        
3512        private void analyzePlsqlTableTypeDefStmt(TPlsqlTableTypeDefStmt stmt) {
3513                TTypeName typeName = stmt.getElementDataType();
3514                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
3515                        Variable cursorVariable = modelFactory.createVariable(stmt.getTypeName());
3516                        cursorVariable.setSubType(SubType.record_type);
3517
3518                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
3519                        if(!variableTable.isCreateTable()) {
3520                                TObjectName starColumn1 = new TObjectName();
3521                                starColumn1.setString("*");
3522                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
3523                                variableTableStarColumn.setShowStar(false);
3524                                variableTableStarColumn.setExpandStar(true);
3525
3526                                TObjectName starColumn = new TObjectName();
3527                                starColumn.setString("*");
3528                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
3529                                variableProperty.setShowStar(false);
3530                                variableProperty.setExpandStar(true);
3531
3532                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
3533                                dataflowRelation.setEffectType(EffectType.rowtype);
3534                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
3535                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
3536                        } else {
3537                                for (TableColumn sourceColumn : variableTable.getColumns()) {
3538                                        String columnName = sourceColumn.getName();
3539                                        TObjectName targetColumn = new TObjectName();
3540                                        targetColumn.setString(columnName);
3541                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
3542                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
3543                                        dataflowRelation.setEffectType(EffectType.rowtype);
3544                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
3545                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
3546                                }
3547                        }
3548                } 
3549        }
3550
3551        private void analyzeMssqlCreateType(TMssqlCreateType createType) {
3552
3553        }
3554
3555        private void analyzeMssqlExecute(TMssqlExecute executeStmt) {
3556                if (executeStmt.getModuleName() != null) {
3557                        TObjectName module = executeStmt.getModuleName();
3558                        if(module.toString().toLowerCase().endsWith("sp_rename")) {
3559                                String oldTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString());        
3560                                Table oldNameTableModel = modelFactory.createTableByName(oldTableName, true);
3561                                List<String> oldTableNames = SQLUtil.parseNames(oldNameTableModel.getName());
3562                                TObjectName oldStarColumn = new TObjectName();
3563                                oldStarColumn.setString("*");
3564                                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
3565                                
3566                                String newTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(1).toString());
3567                                List<String> newTableNames = SQLUtil.parseNames(newTableName);
3568                                if (oldTableNames.size() > newTableNames.size()) {
3569                                        for (int i = oldTableNames.size() - newTableNames.size() - 1; i >= 0; i--) {
3570                                                newTableName = (oldTableNames.get(i) + ".") + newTableName;
3571                                        }
3572                                }
3573
3574                                Table newNameTableModel = modelFactory.createTableByName(newTableName, true);
3575                                TObjectName newStarColumn = new TObjectName();
3576                                newStarColumn.setString("*");
3577                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
3578                                
3579                                Process process = modelFactory.createProcess(executeStmt);
3580                                newNameTableModel.addProcess(process);
3581                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
3582                                relation.setEffectType(EffectType.rename_table);
3583                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
3584                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
3585                                relation.setProcess(process);
3586                                
3587                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
3588                                        oldTableStarColumn.setShowStar(false);
3589                                        relation.setShowStarRelation(false);
3590                                }
3591
3592                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
3593                                        newTableStarColumn.setShowStar(false);
3594                                        relation.setShowStarRelation(false);
3595                                }
3596                        }
3597                        else if(module.toString().toLowerCase().endsWith("sp_executesql")) {
3598                                String sql = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString()); 
3599                                if(sql.startsWith("N")) {
3600                                        sql =  SQLUtil.trimColumnStringQuote(sql.substring(1));
3601                                }
3602                                executeDynamicSql(sql);
3603                        }
3604                        else {
3605                                int argumentSize = executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size();
3606                                String procedureNameWithArgSize = module.toString() + "(" + argumentSize + ")";
3607                                if (argumentSize <= 0  || !DlineageUtil.supportFunctionOverride(option.getVendor())) {
3608                                        procedureNameWithArgSize = module.toString();
3609                                }
3610                                if (procedureDDLMap.containsKey(procedureNameWithArgSize)) {
3611                                        analyzeCustomSqlStmt(procedureDDLMap.get(procedureNameWithArgSize));
3612                                }
3613                                Procedure procedure = modelFactory.createProcedureByName(module, executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size());
3614                                String procedureParent = getProcedureParentName(executeStmt);
3615                                if (procedureParent != null) {
3616                                        Procedure caller = modelManager
3617                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3618                                        if (caller != null) {
3619                                                CallRelationship callRelation = modelFactory.createCallRelation();
3620                                                callRelation.setCallObject(executeStmt);
3621                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3622                                                callRelation.addSource(new ProcedureRelationshipElement(procedure));
3623                                                if(isBuiltInFunctionName(module) || isKeyword(module)){
3624                                                        callRelation.setBuiltIn(true);
3625                                                }
3626                                        }
3627                                }
3628
3629                                if (procedure.getArguments() != null) {
3630                                        for (int i = 0; i < procedure.getArguments().size(); i++) {
3631                                                Argument argument = procedure.getArguments().get(i);
3632                                                Variable variable = modelFactory.createVariable(procedure, argument.getName(), false);
3633                                                if (variable != null) {
3634                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3635                                                                Transform transform = new Transform();
3636                                                                transform.setType(Transform.FUNCTION);
3637                                                                transform.setCode(module);
3638                                                                variable.getColumns().get(0).setTransform(transform);
3639                                                        }
3640                                                        Process process = modelFactory.createProcess(executeStmt);
3641                                                        variable.addProcess(process);
3642                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), executeStmt, argument.getName(), i, process);
3643                                                }
3644                                        }
3645                                }
3646                        }
3647                }
3648        }
3649
3650        private void analyzeDropTableStmt(TDropTableSqlStatement stmt) {
3651                TTable dropTable = stmt.getTargetTable();
3652                if(dropTable == null) {
3653                        return;
3654                }
3655                Table tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(dropTable.getTableName().toString()));
3656                if(tableModel!=null) {
3657                        modelManager.dropTable(tableModel);
3658                }
3659                
3660                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
3661                        tableModel = modelFactory.createTable(dropTable);
3662                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
3663                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
3664                        crudRelationship.setEffectType(EffectType.drop_table);
3665                }
3666        }
3667        
3668        private void analyzeTruncateTableStmt(TTruncateStatement stmt) {
3669                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
3670                        TObjectName table = stmt.getTableName();
3671                        Table tableModel = modelFactory.createTableByName(table);
3672                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
3673                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
3674                        crudRelationship.setEffectType(EffectType.truncate_table);
3675                }
3676        }
3677
3678        private void analyzeCallStmt(TCallStatement callStmt) {
3679                if (callStmt.getRoutineExpr() != null && callStmt.getRoutineExpr().getFunctionCall() != null) {
3680
3681                        TFunctionCall functionCall = callStmt.getRoutineExpr().getFunctionCall();
3682                        Procedure callee = modelManager.getProcedureByName(
3683                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3684                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3685                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3686                                callee = modelManager.getProcedureByName(DlineageUtil
3687                                                .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3688                        }
3689                        if (callee != null) {
3690                                String procedureParent = getProcedureParentName(callStmt);
3691                                if (procedureParent != null) {
3692                                        Procedure caller = modelManager
3693                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3694                                        if (caller != null) {
3695                                                CallRelationship callRelation = modelFactory.createCallRelation();
3696                                                callRelation.setCallObject(callStmt);
3697                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3698                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3699                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
3700                                                                || isKeyword(functionCall.getFunctionName())) {
3701                                                        callRelation.setBuiltIn(true);
3702                                                }
3703                                        }
3704                                }
3705                                if (callee.getArguments() != null) {
3706                                        for (int i = 0; i < callee.getArguments().size(); i++) {
3707                                                Argument argument = callee.getArguments().get(i);
3708                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3709                                                if (variable != null) {
3710                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3711                                                                Transform transform = new Transform();
3712                                                                transform.setType(Transform.FUNCTION);
3713                                                                transform.setCode(functionCall);
3714                                                                variable.getColumns().get(0).setTransform(transform);
3715                                                        }
3716                                                        Process process = modelFactory.createProcess(callStmt);
3717                                                        variable.addProcess(process);
3718                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i,
3719                                                                        process);
3720                                                }
3721                                        }
3722                                }
3723                        } else {
3724                                Function function = (Function) createFunction(functionCall);
3725                                String procedureParent = getProcedureParentName(callStmt);
3726                                if (procedureParent != null) {
3727                                        Procedure caller = modelManager
3728                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3729                                        if (caller != null) {
3730                                                CallRelationship callRelation = modelFactory.createCallRelation();
3731                                                callRelation.setCallObject(callStmt);
3732                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3733                                                callRelation.addSource(new FunctionRelationshipElement(function));
3734                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
3735                                                                || isKeyword(functionCall.getFunctionName())) {
3736                                                        callRelation.setBuiltIn(true);
3737                                                }
3738                                        }
3739                                }
3740                        }
3741                }
3742                else if (callStmt.getRoutineName() != null) {
3743                        TObjectName function = callStmt.getRoutineName();
3744                        String functionName = function.toString();
3745                        Procedure callee = modelManager.getProcedureByName(
3746                                        DlineageUtil.getIdentifierNormalTableName(functionName));
3747                        if (callee == null && procedureDDLMap.containsKey(functionName)) {
3748                                analyzeCustomSqlStmt(procedureDDLMap.get(functionName));
3749                                callee = modelManager.getProcedureByName(functionName);
3750                        }
3751                        if (callee != null) {
3752                                String procedureParent = getProcedureParentName(callStmt);
3753                                if (procedureParent != null) {
3754                                        Procedure caller = modelManager
3755                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3756                                        if (caller != null) {
3757                                                CallRelationship callRelation = modelFactory.createCallRelation();
3758                                                callRelation.setCallObject(callStmt);
3759                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3760                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3761                                                if (isBuiltInFunctionName(function)
3762                                                                || isKeyword(function)) {
3763                                                        callRelation.setBuiltIn(true);
3764                                                }
3765                                        }
3766                                }
3767                                if (callee.getArguments() != null) {
3768                                        for (int i = 0; i < callee.getArguments().size(); i++) {
3769                                                Argument argument = callee.getArguments().get(i);
3770                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3771                                                if (variable != null) {
3772                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3773                                                                Transform transform = new Transform();
3774                                                                transform.setType(Transform.FUNCTION);
3775                                                                transform.setCode(callStmt);
3776                                                                variable.getColumns().get(0).setTransform(transform);
3777                                                        }
3778                                                        Process process = modelFactory.createProcess(callStmt);
3779                                                        variable.addProcess(process);
3780                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), callStmt, i,
3781                                                                        process);
3782                                                }
3783                                        }
3784                                }
3785                        }
3786                }
3787        }
3788
3789        private boolean analyzeCustomFunctionCall(TFunctionCall functionCall) {
3790                Procedure callee = modelManager.getProcedureByName(
3791                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3792                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3793                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3794                        callee = modelManager.getProcedureByName(
3795                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3796                }
3797                if (callee != null) {
3798                        String procedureParent = getProcedureParentName(stmtStack.peek());
3799                        if (procedureParent != null) {
3800                                Procedure caller = modelManager
3801                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3802                                if (caller != null) {
3803                                        CallRelationship callRelation = modelFactory.createCallRelation();
3804                                        callRelation.setCallObject(functionCall);
3805                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
3806                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
3807                                        if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
3808                                                callRelation.setBuiltIn(true);
3809                                        }
3810                                }
3811                        }
3812                        if (callee.getArguments() != null) {
3813                                for (int i = 0; i < callee.getArguments().size(); i++) {
3814                                        Argument argument = callee.getArguments().get(i);
3815                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3816                                        if(variable!=null) {
3817                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3818                                                        Transform transform = new Transform();
3819                                                        transform.setType(Transform.FUNCTION);
3820                                                        transform.setCode(functionCall);
3821                                                        variable.getColumns().get(0).setTransform(transform);
3822                                                }
3823                                                Process process = modelFactory.createProcess(functionCall);
3824                                                variable.addProcess(process);
3825                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i, process);
3826                                        }
3827                                }
3828                        }
3829                        return true;
3830                }
3831                return false;
3832        }
3833
3834        private void analyzeOracleBasicStmt(TBasicStmt oracleBasicStmt) {
3835                if (oracleBasicStmt.getExpr() == null || oracleBasicStmt.getExpr().getFunctionCall() == null) {
3836                        return;
3837                }
3838
3839                TFunctionCall functionCall = oracleBasicStmt.getExpr().getFunctionCall();
3840                Procedure callee = modelManager.getProcedureByName(
3841                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3842                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3843                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3844                        callee = modelManager.getProcedureByName(
3845                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3846                }
3847                if (callee != null) {
3848                        String procedureParent = getProcedureParentName(oracleBasicStmt);
3849                        if (procedureParent != null) {
3850                                Procedure caller = modelManager
3851                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3852                                if (caller != null) {
3853                                        CallRelationship callRelation = modelFactory.createCallRelation();
3854                                        callRelation.setCallObject(functionCall);
3855                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
3856                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
3857                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
3858                                                        || (isBuiltInFunctionName(functionCall.getFunctionName())
3859                                                        || isKeyword(functionCall.getFunctionName()))) {
3860                                                callRelation.setBuiltIn(true);
3861                                        }
3862                                }
3863                        }
3864                        if (callee.getArguments() != null) {
3865                                for (int i = 0; i < callee.getArguments().size(); i++) {
3866                                        Argument argument = callee.getArguments().get(i);
3867                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3868                                        if(variable!=null) {
3869                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3870                                                        Transform transform = new Transform();
3871                                                        transform.setType(Transform.FUNCTION);
3872                                                        transform.setCode(functionCall);
3873                                                        variable.getColumns().get(0).setTransform(transform);
3874                                                }
3875                                                Process process = modelFactory.createProcess(functionCall);
3876                                                variable.addProcess(process);
3877                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i, process);
3878                                        }
3879                                }
3880                        }
3881                } else {
3882                        Function function = modelFactory.createFunction(functionCall);
3883                        String procedureParent = getProcedureParentName(oracleBasicStmt);
3884                        if (procedureParent != null) {
3885                                Procedure caller = modelManager
3886                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3887                                if (caller != null) {
3888                                        CallRelationship callRelation = modelFactory.createCallRelation();
3889                                        callRelation.setCallObject(functionCall);
3890                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
3891                                        callRelation.addSource(new FunctionRelationshipElement(function));
3892                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
3893                                                        || (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName()))) {
3894                                                callRelation.setBuiltIn(true);
3895                                        }
3896                                }
3897                        }
3898                }
3899        }
3900
3901        private void analyzeIfStmt(TIfStmt ifStmt) {
3902                if (ifStmt.getCondition() != null) {
3903                        columnsInExpr visitor = new columnsInExpr();
3904                        ifStmt.getCondition().inOrderTraverse(visitor);
3905                        List<TParseTreeNode> functions = visitor.getFunctions();
3906
3907                        if (functions != null && !functions.isEmpty()) {
3908                                for (int i = 0; i < functions.size(); i++) {
3909                                        if (!(functions.get(i) instanceof TFunctionCall))
3910                                                continue;
3911                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
3912                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
3913                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3914                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3915                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3916                                                callee = modelManager.getProcedureByName(
3917                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3918                                        }
3919                                        if (callee != null) {
3920                                                String procedureParent = getProcedureParentName(ifStmt);
3921                                                if (procedureParent != null) {
3922                                                        Procedure caller = modelManager
3923                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3924                                                        if (caller != null) {
3925                                                                CallRelationship callRelation = modelFactory.createCallRelation();
3926                                                                callRelation.setCallObject(functionCall);
3927                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3928                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3929                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
3930                                                                        callRelation.setBuiltIn(true);
3931                                                                }
3932                                                        }
3933                                                }
3934                                                if (callee.getArguments() != null) {
3935                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
3936                                                                Argument argument = callee.getArguments().get(j);
3937                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3938                                                                if(variable!=null) {
3939                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3940                                                                                Transform transform = new Transform();
3941                                                                                transform.setType(Transform.FUNCTION);
3942                                                                                transform.setCode(functionCall);
3943                                                                                variable.getColumns().get(0).setTransform(transform);
3944                                                                        }
3945                                                                        Process process = modelFactory.createProcess(functionCall);
3946                                                                        variable.addProcess(process);
3947                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
3948                                                                }
3949                                                        }
3950                                                }
3951                                        } else {
3952                                                Function function = modelFactory.createFunction(functionCall);
3953                                                String procedureParent = getProcedureParentName(ifStmt);
3954                                                if (procedureParent != null) {
3955                                                        Procedure caller = modelManager
3956                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3957                                                        if (caller != null) {
3958                                                                CallRelationship callRelation = modelFactory.createCallRelation();
3959                                                                callRelation.setCallObject(functionCall);
3960                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3961                                                                callRelation.addSource(new FunctionRelationshipElement(function));
3962                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
3963                                                                        callRelation.setBuiltIn(true);
3964                                                                }
3965                                                        }
3966                                                }
3967                                        }
3968                                }
3969                        }
3970                }
3971
3972                if (ifStmt.getThenStatements() != null) {
3973                        for (int i = 0; i < ifStmt.getThenStatements().size(); ++i) {
3974                                analyzeCustomSqlStmt(ifStmt.getThenStatements().get(i));
3975                                ResultSet returnResult = modelFactory.createResultSet(ifStmt.getThenStatements().get(i), false);
3976                                if(returnResult!=null){
3977                                        for(ResultColumn resultColumn: returnResult.getColumns()){
3978                                                analyzeFilterCondition(resultColumn, ifStmt.getCondition(), null, null, EffectType.function);
3979                                        }
3980                                }
3981                        }
3982                }
3983
3984                if (ifStmt.getElseifStatements() != null) {
3985                        for (int i = 0; i < ifStmt.getElseifStatements().size(); ++i) {
3986                                analyzeCustomSqlStmt(ifStmt.getElseifStatements().get(i));
3987                        }
3988                }
3989
3990                if (ifStmt.getElseStatements() != null) {
3991                        for (int i = 0; i < ifStmt.getElseStatements().size(); ++i) {
3992                                analyzeCustomSqlStmt(ifStmt.getElseStatements().get(i));
3993                        }
3994                }
3995        }
3996
3997        private void analyzeElsIfStmt(TElsifStmt elsIfStmt) {
3998                if (elsIfStmt.getCondition() != null) {
3999                        columnsInExpr visitor = new columnsInExpr();
4000                        elsIfStmt.getCondition().inOrderTraverse(visitor);
4001                        List<TParseTreeNode> functions = visitor.getFunctions();
4002
4003                        if (functions != null && !functions.isEmpty()) {
4004                                for (int i = 0; i < functions.size(); i++) {
4005                                        if (!(functions.get(i) instanceof TFunctionCall))
4006                                                continue;
4007                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
4008                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
4009                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4010                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
4011                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4012                                                callee = modelManager.getProcedureByName(
4013                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4014                                        }
4015                                        if (callee != null) {
4016                                                String procedureParent = getProcedureParentName(elsIfStmt);
4017                                                if (procedureParent != null) {
4018                                                        Procedure caller = modelManager
4019                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4020                                                        if (caller != null) {
4021                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4022                                                                callRelation.setCallObject(functionCall);
4023                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4024                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
4025                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
4026                                                                        callRelation.setBuiltIn(true);
4027                                                                }
4028                                                        }
4029                                                }
4030                                                if (callee.getArguments() != null) {
4031                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
4032                                                                Argument argument = callee.getArguments().get(j);
4033                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4034                                                                if(variable!=null) {
4035                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4036                                                                                Transform transform = new Transform();
4037                                                                                transform.setType(Transform.FUNCTION);
4038                                                                                transform.setCode(functionCall);
4039                                                                                variable.getColumns().get(0).setTransform(transform);
4040                                                                        }
4041                                                                        Process process = modelFactory.createProcess(functionCall);
4042                                                                        variable.addProcess(process);
4043                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
4044                                                                }
4045                                                        }
4046                                                }
4047                                        } else {
4048                                                Function function = modelFactory.createFunction(functionCall);
4049                                                String procedureParent = getProcedureParentName(elsIfStmt);
4050                                                if (procedureParent != null) {
4051                                                        Procedure caller = modelManager
4052                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4053                                                        if (caller != null) {
4054                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4055                                                                callRelation.setCallObject(functionCall);
4056                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4057                                                                callRelation.addSource(new FunctionRelationshipElement(function));
4058                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
4059                                                                        callRelation.setBuiltIn(true);
4060                                                                }
4061                                                        }
4062                                                }
4063                                        }
4064                                }
4065                        }
4066                }
4067
4068                if (elsIfStmt.getThenStatements() != null) {
4069                        for (int i = 0; i < elsIfStmt.getThenStatements().size(); ++i) {
4070                                ResultSet returnResult = modelFactory.createResultSet(elsIfStmt.getThenStatements().get(i), false);
4071                                if (returnResult != null) {
4072                                        for (ResultColumn resultColumn : returnResult.getColumns()) {
4073                                                analyzeFilterCondition(resultColumn, elsIfStmt.getCondition(), null, null, EffectType.function);
4074                                        }
4075                                }
4076                                analyzeCustomSqlStmt(elsIfStmt.getThenStatements().get(i));
4077                        }
4078                }
4079        }
4080
4081        private void analyzeCloneTableStmt(TCreateTableSqlStatement stmt) {
4082                if (stmt.getCloneSourceTable() != null) {
4083                        Table sourceTable = modelFactory.createTableByName(stmt.getCloneSourceTable());
4084                        Table cloneTable = modelFactory.createTableByName(stmt.getTableName());
4085                        Process process = modelFactory.createProcess(stmt);
4086                        cloneTable.addProcess(process);
4087
4088                        if (sourceTable.isDetermined()) {
4089                                for (int k = 0; k < sourceTable.getColumns().size(); k++) {
4090                                        TableColumn sourceColumn = sourceTable.getColumns().get(k);
4091                                        TObjectName objectName = new TObjectName();
4092                                        objectName.setString(sourceColumn.getName());
4093                                        TableColumn tableColumn = modelFactory.createTableColumn(cloneTable, objectName, true);
4094                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4095                                        dataflowRelation.setEffectType(EffectType.clone_table);
4096                                        dataflowRelation
4097                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
4098                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
4099                                        dataflowRelation.setProcess(process);
4100                                }
4101                                cloneTable.setDetermined(true);
4102                                cloneTable.setFromDDL(true);
4103                        } else {
4104                                TObjectName sourceName = new TObjectName();
4105                                sourceName.setString("*");
4106                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceName, false);
4107                                TObjectName targetName = new TObjectName();
4108                                targetName.setString("*");
4109                                TableColumn targetTableColumn = modelFactory.createTableColumn(cloneTable, targetName, false);
4110                                if(sourceTableColumn!=null && targetTableColumn!=null) {
4111                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4112                                        dataflowRelation.setEffectType(EffectType.clone_table);
4113                                        dataflowRelation
4114                                                        .addSource(new TableColumnRelationshipElement(sourceTableColumn));
4115                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
4116                                        dataflowRelation.setProcess(process);
4117                                }
4118                        }
4119                }
4120        }
4121
4122        private void analyzeCloneDatabaseStmt(TCreateDatabaseSqlStatement stmt) {
4123                if (stmt.getCloneSourceDb() != null) {
4124                        Database sourceDatabase = modelFactory.createDatabase(stmt.getCloneSourceDb());
4125                        Database cloneDatabase = modelFactory.createDatabase(stmt.getDatabaseName());
4126                        Process process = modelFactory.createProcess(stmt);
4127                        cloneDatabase.addProcess(process);
4128                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4129                        relation.setEffectType(EffectType.clone_database);
4130                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneDatabase.getRelationRows()));
4131                        relation.addSource(
4132                                        new RelationRowsRelationshipElement<TableRelationRows>(sourceDatabase.getRelationRows()));
4133                        relation.setProcess(process);
4134                }
4135        }
4136
4137        private void analyzeCloneSchemaStmt(TCreateSchemaSqlStatement stmt) {
4138                if (stmt.getCloneSourceSchema() != null) {
4139                        Schema sourceSchema = modelFactory.createSchema(stmt.getCloneSourceSchema());
4140                        Schema cloneSchema = modelFactory.createSchema(stmt.getSchemaName());
4141                        Process process = modelFactory.createProcess(stmt);
4142                        cloneSchema.addProcess(process);
4143                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4144                        relation.setEffectType(EffectType.clone_schema);
4145                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneSchema.getRelationRows()));
4146                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(sourceSchema.getRelationRows()));
4147                        relation.setProcess(process);
4148                }
4149        }
4150
4151        private void executeDynamicSql(String sql) {
4152                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
4153                sqlparser.sqltext = sql;
4154                int result = sqlparser.parse();
4155                if (result == 0) {
4156                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4157                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4158                        }
4159                }
4160        }
4161
4162        private void extractSnowflakeSQLFromProcedure(TCreateProcedureStmt procedure) {
4163                Map<String, String> argMap = new LinkedHashMap<String, String>();
4164                if (procedure.getParameterDeclarations() != null) {
4165                        for (int i = 0; i < procedure.getParameterDeclarations().size(); i++) {
4166                                TParameterDeclaration def = procedure.getParameterDeclarations().getParameterDeclarationItem(i);
4167                                argMap.put(def.getParameterName().toString(), def.getDataType().getDataTypeName());
4168                        }
4169                }
4170                StringBuilder buffer = new StringBuilder();
4171                buffer.append("(function(");
4172                String[] args = argMap.keySet().toArray(new String[0]);
4173                for (int i = 0; i < args.length; i++) {
4174                        buffer.append(args[i].toUpperCase());
4175                        if (i < args.length - 1) {
4176                                buffer.append(",");
4177                        }
4178                }
4179                buffer.append("){\n");
4180
4181                int start = -1;
4182                int end = -1;
4183                boolean dollar = false;
4184                boolean quote = false;
4185                if (procedure.getRoutineBody().indexOf("$$") != -1) {
4186                        start = procedure.getRoutineBody().indexOf("$$") + 2;
4187                        end = procedure.getRoutineBody().lastIndexOf("$$") - 1;
4188                        dollar = true;
4189                } else if (procedure.getRoutineBody().indexOf("'") != -1) {
4190                        start = procedure.getRoutineBody().indexOf("'") + 1;
4191                        end = procedure.getRoutineBody().lastIndexOf("'");
4192                        quote = true;
4193                }
4194                String body = procedure.getRoutineBody().substring(start, end);
4195                if (dollar && body.indexOf("`") != -1) {
4196                        Pattern pattern = Pattern.compile("`.+?`", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
4197                        Matcher matcher = pattern.matcher(body);
4198                        StringBuffer replaceBuffer = new StringBuffer();
4199                        while (matcher.find()) {
4200                                String condition = matcher.group().replace("\r\n", "\n").replace("'", "\\\\'")
4201                                                .replace("\n", "\\\\n'\n+'").replace("`", "'").replace("$", "RDS_CHAR_DOLLAR");
4202                                matcher.appendReplacement(replaceBuffer, condition);
4203                        }
4204                        matcher.appendTail(replaceBuffer);
4205                        body = replaceBuffer.toString().replace("RDS_CHAR_DOLLAR", "$");
4206                }
4207                if (quote && body.indexOf("'") != -1) {
4208                        body = body.replace("''", "'");
4209                }
4210                buffer.append(body);
4211                buffer.append("})(");
4212                for (int i = 0; i < args.length; i++) {
4213                        String type = argMap.get(args[i]);
4214                        if (type.equalsIgnoreCase("VARCHAR")) {
4215                                buffer.append("'pseudo'");
4216                        } else if (type.equalsIgnoreCase("STRING")) {
4217                                buffer.append("'pseudo'");
4218                        } else if (type.equalsIgnoreCase("CHAR")) {
4219                                buffer.append("'pseudo'");
4220                        } else if (type.equalsIgnoreCase("CHARACTER")) {
4221                                buffer.append("'pseudo'");
4222                        } else if (type.equalsIgnoreCase("TEXT")) {
4223                                buffer.append("'pseudo'");
4224                        } else if (type.equalsIgnoreCase("BINARY")) {
4225                                buffer.append("'pseudo'");
4226                        } else if (type.equalsIgnoreCase("VARBINARY")) {
4227                                buffer.append("'pseudo'");
4228                        } else if (type.equalsIgnoreCase("BOOLEAN")) {
4229                                buffer.append(true);
4230                        } else if (type.equalsIgnoreCase("FLOAT")) {
4231                                buffer.append("1.0");
4232                        } else if (type.equalsIgnoreCase("FLOAT4")) {
4233                                buffer.append("1.0");
4234                        } else if (type.equalsIgnoreCase("FLOAT8")) {
4235                                buffer.append("1.0");
4236                        } else if (type.equalsIgnoreCase("DOUBLE")) {
4237                                buffer.append("1.0");
4238                        } else if (type.equalsIgnoreCase("DOUBLE PRECISION")) {
4239                                buffer.append("1.0");
4240                        } else if (type.equalsIgnoreCase("REAL")) {
4241                                buffer.append("1.0");
4242                        } else if (type.equalsIgnoreCase("NUMBER")) {
4243                                buffer.append("1.0");
4244                        } else if (type.equalsIgnoreCase("DECIMAL")) {
4245                                buffer.append("1.0");
4246                        } else if (type.equalsIgnoreCase("NUMERIC")) {
4247                                buffer.append("1.0");
4248                        } else if (type.equalsIgnoreCase("INT")) {
4249                                buffer.append("1");
4250                        } else if (type.equalsIgnoreCase("INTEGER")) {
4251                                buffer.append("1");
4252                        } else if (type.equalsIgnoreCase("BIGINT")) {
4253                                buffer.append("1");
4254                        } else if (type.equalsIgnoreCase("SMALLINT")) {
4255                                buffer.append("1");
4256                        } else if (type.equalsIgnoreCase("DATE")) {
4257                                buffer.append("new Date()");
4258                        } else if (type.equalsIgnoreCase("DATETIME")) {
4259                                buffer.append("new Date()");
4260                        } else if (type.equalsIgnoreCase("TIME")) {
4261                                buffer.append("new Date()");
4262                        } else if (type.equalsIgnoreCase("TIMESTAMP")) {
4263                                buffer.append("new Date()");
4264                        } else if (type.equalsIgnoreCase("TIMESTAMP_LTZ")) {
4265                                buffer.append("new Date()");
4266                        } else if (type.equalsIgnoreCase("TIMESTAMP_NTZ")) {
4267                                buffer.append("new Date()");
4268                        } else if (type.equalsIgnoreCase("TIMESTAMP_TZ")) {
4269                                buffer.append("new Date()");
4270                        } else if (type.equalsIgnoreCase("VARIANT")) {
4271                                buffer.append("{}");
4272                        } else if (type.equalsIgnoreCase("OBJECT")) {
4273                                buffer.append("{}");
4274                        } else if (type.equalsIgnoreCase("ARRAY")) {
4275                                buffer.append("[]");
4276                        } else if (type.equalsIgnoreCase("GEOGRAPHY")) {
4277                                buffer.append("{}");
4278                        }
4279                        if (i < args.length - 1) {
4280                                buffer.append(",");
4281                        }
4282                }
4283                buffer.append(");");
4284
4285                try {
4286                        ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
4287                        ScriptEngine nashorn = scriptEngineManager.getEngineByName("nashorn");
4288                        nashorn.put("analyzer", this);
4289                        nashorn.eval(new InputStreamReader(
4290                                        getClass().getResourceAsStream("/gudusoft/gsqlparser/parser/snowflake/snowflake.js")));
4291                        nashorn.eval(new StringReader(buffer.toString()));
4292                } catch (ScriptException e) {
4293                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
4294                        sqlparser.sqltext = body;
4295                        int result = sqlparser.parse();
4296                        if (result == 0) {
4297                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4298                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4299                                }
4300                                return;
4301                        }
4302                        ErrorInfo errorInfo = new ErrorInfo();
4303                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
4304                        errorInfo.setErrorMessage("Invoke script error: " + e.getMessage());
4305                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(procedure.getStartToken().lineNo,
4306                                        procedure.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
4307                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(procedure.getEndToken().lineNo,
4308                                        procedure.getEndToken().columnNo + procedure.getEndToken().getAstext().length(),
4309                                        ModelBindingManager.getGlobalHash()));
4310                        errorInfos.add(errorInfo);
4311                }
4312        }
4313
4314        private void analyzeHiveLoadStmt(THiveLoad stmt) {
4315                if (stmt.getPath() != null && stmt.getTable() != null) {
4316                        Table uriFile = modelFactory.createTableByName(stmt.getPath(), true);
4317                        uriFile.setPath(true);
4318                        uriFile.setCreateTable(true);
4319                        TObjectName fileUri = new TObjectName();
4320                        fileUri.setString("uri=" + stmt.getPath());
4321                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
4322
4323                        Table tableModel = modelFactory.createTable(stmt.getTable());
4324                        Process process = modelFactory.createProcess(stmt);
4325                        tableModel.addProcess(process);
4326
4327                        TPartitionExtensionClause p = stmt.getTable().getPartitionExtensionClause();
4328                        if (p.getKeyValues() != null && p.getKeyValues().size() > 0) {
4329                                for (int i = 0; i < p.getKeyValues().size(); i++) {
4330                                        TExpression expression = p.getKeyValues().getExpression(i);
4331                                        if (expression.getLeftOperand().getExpressionType() == EExpressionType.simple_object_name_t) {
4332                                                modelFactory.createTableColumn(tableModel, expression.getLeftOperand().getObjectOperand(),
4333                                                                true);
4334                                        }
4335                                }
4336                        }
4337
4338                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
4339                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4340                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4341                                relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
4342                                relation.setProcess(process);
4343                        }
4344                }
4345        }
4346        
4347        private void analyzeLoadDataStmt(TLoadDataStmt stmt) {
4348                
4349        }
4350
4351        private void analyzeUnloadStmt(TUnloadStmt unloadStmt) {
4352                if (unloadStmt.getSelectSqlStatement() != null && unloadStmt.getS3() != null) {
4353
4354                        Table uriFile = modelFactory.createTableByName(unloadStmt.getS3(), true);
4355                        uriFile.setPath(true);
4356                        uriFile.setCreateTable(true);
4357                        TObjectName fileUri = new TObjectName();
4358                        fileUri.setString("uri=" + unloadStmt.getS3());
4359                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
4360
4361                        Process process = modelFactory.createProcess(unloadStmt);
4362                        uriFile.addProcess(process);
4363
4364                        TCustomSqlStatement stmt = unloadStmt.getSelectSqlStatement();
4365                        analyzeCustomSqlStmt(stmt);
4366                        if (stmt instanceof TSelectSqlStatement) {
4367                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
4368                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(select);
4369                                if (resultSetModel != null) {
4370                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
4371                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
4372                                                if (resultColumn.hasStarLinkColumn()
4373                                                                && resultColumn.getStarLinkColumnNames().size() > 0) {
4374                                                        for (int k = 0; k < resultColumn.getStarLinkColumnNames().size(); k++) {
4375                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(resultSetModel,
4376                                                                                resultColumn.getStarLinkColumnName(k), false);
4377                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4378                                                                dataflowRelation.setEffectType(EffectType.unload);
4379                                                                dataflowRelation
4380                                                                                .addSource(new ResultColumnRelationshipElement(expandStarColumn));
4381                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
4382                                                                dataflowRelation.setProcess(process);
4383                                                        }
4384                                                }
4385                                                if (!resultColumn.hasStarLinkColumn() || resultColumn.isShowStar()) {
4386                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4387                                                        dataflowRelation.setEffectType(EffectType.unload);
4388                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
4389                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
4390                                                        dataflowRelation.setProcess(process);
4391                                                }
4392                                        }
4393                                }
4394                        }
4395                }
4396
4397        }
4398
4399        private void analyzeCopyIntoStmt(TSnowflakeCopyIntoStmt stmt) {
4400                if (stmt.getTableName() != null) {
4401                        if (stmt.getStageLocation() != null) {
4402                                Table intoTable = modelManager
4403                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4404                                if (intoTable == null) {
4405                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4406                                        TObjectName starColumn = new TObjectName();
4407                                        starColumn.setString("*");
4408                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4409                                        if (column != null) {
4410                                                column.setExpandStar(false);
4411                                                column.setPseduo(true);
4412                                        }
4413                                }
4414                                Process process = modelFactory.createProcess(stmt);
4415                                intoTable.addProcess(process);
4416
4417                                TObjectName stageName = stmt.getStageLocation().getStageName();
4418                                if (stageName == null || stmt.getStageLocation().getTableName() != null) {
4419                                        stageName = stmt.getStageLocation().getTableName();
4420                                }
4421                                if (stageName != null) {
4422                                        String stageFullName = DlineageUtil.getTableFullName(stageName.toString());
4423                                        Table stage = modelManager.getTableByName(stageFullName);
4424                                        if (stage == null) {
4425                                                stage = modelFactory.createStage(stageName);
4426                                                stage.setCreateTable(true);
4427                                                stage.setStage(true);
4428                                                if (stmt.getStageLocation().getPath() != null) {
4429                                                        stage.setLocation(stmt.getStageLocation().getPath().toString());
4430                                                        TObjectName location = new TObjectName();
4431                                                        location.setString(stmt.getStageLocation().getPath().toString());
4432                                                        modelFactory.createStageLocation(stage, location);
4433                                                } else {
4434                                                        stage.setLocation("unknownPath");
4435                                                        TObjectName location = new TObjectName();
4436                                                        location.setString("unknownPath");
4437                                                        modelFactory.createStageLocation(stage, location);
4438                                                }
4439                                        }
4440
4441                                        if (stage != null && intoTable != null) {
4442                                                if (intoTable != null && !intoTable.getColumns().isEmpty() && !stage.getColumns().isEmpty()) {
4443                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
4444                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4445                                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
4446                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4447                                                                relation.setProcess(process);
4448                                                        }
4449                                                }
4450                                        }
4451                                } else if (stmt.getStageLocation().getExternalLocation() != null) {
4452                                        Table pathModel = modelFactory.createTableByName(stmt.getStageLocation().getExternalLocation(),
4453                                                        true);
4454                                        pathModel.setPath(true);
4455                                        pathModel.setCreateTable(true);
4456                                        TableColumn fileUriColumn = modelFactory.createFileUri(pathModel,
4457                                                        stmt.getStageLocation().getExternalLocation());
4458                                        if (intoTable != null) {
4459                                                if (intoTable != null && !intoTable.getColumns().isEmpty()) {
4460                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
4461                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4462                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4463                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4464                                                                relation.setProcess(process);
4465                                                        }
4466                                                }
4467                                        }
4468                                }
4469                        }
4470                        else if (stmt.getSubQuery() != null) {
4471                                analyzeSelectStmt(stmt.getSubQuery());
4472                                
4473                                Table intoTable = modelManager
4474                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4475                                if (intoTable == null) {
4476                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4477                                        TObjectName starColumn = new TObjectName();
4478                                        starColumn.setString("*");
4479                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4480                                        column.setExpandStar(true);
4481                                        column.setPseduo(true);
4482                                        
4483                                        Process process = modelFactory.createProcess(stmt);
4484                                        intoTable.addProcess(process);
4485                                        
4486                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
4487                                        if (resultSetModel != null) {
4488                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
4489                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
4490                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4491                                                        dataflowRelation.setEffectType(EffectType.copy);
4492                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
4493                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(column));
4494                                                }
4495                                        }
4496                                }
4497                        }
4498                }
4499        }
4500
4501        private void analyzeRedshiftCopyStmt(TRedshiftCopy stmt) {
4502                if (stmt.getTableName() != null && stmt.getFromSource() != null) {
4503
4504                        Table intoTable = modelManager
4505                                        .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4506                        if (intoTable == null) {
4507                                intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4508                                if (stmt.getColumnList() == null || stmt.getColumnList().size() == 0) {
4509                                        TObjectName starColumn = new TObjectName();
4510                                        starColumn.setString("*");
4511                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4512                                        if (column != null) {
4513                                                column.setExpandStar(false);
4514                                                column.setPseduo(true);
4515                                        }
4516                                } else {
4517                                        for (TObjectName columnName : stmt.getColumnList()) {
4518                                                modelFactory.createTableColumn(intoTable, columnName, true);
4519                                        }
4520                                }
4521                        }
4522                        Process process = modelFactory.createProcess(stmt);
4523                        intoTable.addProcess(process);
4524
4525                        if (stmt.getFromSource() != null) {
4526                                Table pathModel = modelFactory.createTableByName(stmt.getFromSource(), true);
4527                                pathModel.setPath(true);
4528                                pathModel.setCreateTable(true);
4529                                TObjectName fileUri = new TObjectName();
4530                                fileUri.setString(stmt.getFromSource());
4531                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
4532                                if (intoTable != null) {
4533                                        if (intoTable != null && !intoTable.getColumns().isEmpty()) {
4534                                                for (int i = 0; i < intoTable.getColumns().size(); i++) {
4535                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4536                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4537                                                        relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4538                                                        relation.setProcess(process);
4539                                                }
4540                                        }
4541                                }
4542                        }
4543                }
4544        }
4545
4546        private void analyzeCreateIndexExpressionOperand(TExpression expr, Table tableModel){
4547                if(expr == null) return;
4548                Deque<TExpression> stack = new ArrayDeque<>();
4549                stack.push(expr);
4550                while (!stack.isEmpty()) {
4551                        TExpression current = stack.pop();
4552                        if (current == null) continue;
4553                        TObjectName columnObj = current.getObjectOperand();
4554                        if (columnObj != null) {
4555                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, columnObj, true);
4556                                tableConstraint.setIndexKey(true);
4557                        } else {
4558                                if (current.getRightOperand() != null) {
4559                                        stack.push(current.getRightOperand());
4560                                }
4561                                if (current.getLeftOperand() != null) {
4562                                        stack.push(current.getLeftOperand());
4563                                }
4564                        }
4565                }
4566        }
4567        private void analyzeCreateIndexStageStmt(TCreateIndexSqlStatement stmt) {
4568        if(stmt.getTableName() == null){
4569            return;
4570        }
4571                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
4572                TOrderByItemList columns = stmt.getColumnNameList();
4573        if(columns!=null) {
4574            for (int i = 0; i < columns.size(); i++) {
4575                TExpression expr = columns.getOrderByItem(i).getSortKey();
4576                analyzeCreateIndexExpressionOperand(expr, tableModel);
4577            }
4578        }
4579        }
4580
4581        private void analyzeCreateSynonymStmt(TCreateSynonymStmt stmt) {
4582                TObjectName sourceTableName = stmt.getForName();
4583                if(sourceTableName == null) {
4584                        return;
4585                }
4586                TCustomSqlStatement createView = viewDDLMap
4587                                .get(DlineageUtil.getTableFullName(sourceTableName.toString()));
4588                if (createView != null) {
4589                        analyzeCustomSqlStmt(createView);
4590                }
4591                Table sourceTableModel = modelFactory.createTableByName(sourceTableName);
4592                Process process = modelFactory.createProcess(stmt);
4593                sourceTableModel.addProcess(process);
4594                if(stmt.getSynonymName()!=null) {
4595                        Table synonymTableModel = modelFactory.createTableByName(stmt.getSynonymName());
4596                        synonymTableModel.setSubType(SubType.synonym);
4597                        if(sourceTableModel.isCreateTable()) {
4598                                synonymTableModel.setCreateTable(true);
4599                                for(TableColumn sourceTableColumn: sourceTableModel.getColumns()) {
4600                                        TObjectName columnName = new TObjectName();
4601                                        columnName.setString(sourceTableColumn.getName());
4602                                        TableColumn synonymTableColumn = new TableColumn(synonymTableModel, columnName);
4603                                        synonymTableModel.addColumn(synonymTableColumn);
4604                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4605                                        relation.setEffectType(EffectType.create_synonym);
4606                                        relation.setTarget(new TableColumnRelationshipElement(synonymTableColumn));
4607                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
4608                                        relation.setProcess(process);
4609                                }
4610                        }
4611                        else {
4612                                TObjectName synonymStarColumn = new TObjectName();
4613                                synonymStarColumn.setString("*");
4614                                TableColumn synonymTableStarColumn = modelFactory.createTableColumn(synonymTableModel,
4615                                                synonymStarColumn, true);
4616                                TObjectName sourceStarColumn = new TObjectName();
4617                                sourceStarColumn.setString("*");
4618                                TableColumn sourceTableStarColumn = modelFactory.createTableColumn(sourceTableModel, sourceStarColumn, true);
4619                                
4620                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4621                                relation.setEffectType(EffectType.create_synonym);
4622                                relation.setTarget(new TableColumnRelationshipElement(synonymTableStarColumn));
4623                                relation.addSource(new TableColumnRelationshipElement(sourceTableStarColumn));
4624                                relation.setProcess(process);
4625                        }
4626                }
4627        }
4628        
4629        private void analyzeRenameStmt(TRenameStmt stmt) {
4630                TObjectName oldTableName = stmt.getOldName();
4631                TObjectName newTableName = stmt.getNewName();
4632                
4633                Table oldNameTableModel = modelFactory.createTableByName(oldTableName);
4634                TObjectName oldStarColumn = new TObjectName();
4635                oldStarColumn.setString("*");
4636                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
4637                
4638                Table newNameTableModel = modelFactory.createTableByName(newTableName);
4639                TObjectName newStarColumn = new TObjectName();
4640                newStarColumn.setString("*");
4641                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
4642                
4643                Process process = modelFactory.createProcess(stmt);
4644                newNameTableModel.addProcess(process);
4645                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4646                relation.setEffectType( EffectType.rename_table);
4647                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4648                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4649                relation.setProcess(process);
4650                
4651                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4652                        oldTableStarColumn.setShowStar(false);
4653                        relation.setShowStarRelation(false);
4654                }
4655
4656                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4657                        newTableStarColumn.setShowStar(false);
4658                        relation.setShowStarRelation(false);
4659                }
4660        }
4661        
4662        private void analyzeAlterTableStmt(TAlterTableStatement stmt) {
4663                TTable oldNameTable = stmt.getTargetTable();
4664                if (oldNameTable == null) {
4665                        return;
4666                }
4667
4668                Table oldNameTableModel = modelFactory.createTable(oldNameTable);
4669                TObjectName oldStarColumn = new TObjectName();
4670                oldStarColumn.setString("*");
4671                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
4672
4673                for (int i = 0; stmt.getAlterTableOptionList() != null && i < stmt.getAlterTableOptionList().size(); i++) {
4674                        TAlterTableOption option = stmt.getAlterTableOptionList().getAlterTableOption(i);
4675                        if (option.getOptionType() == EAlterTableOptionType.RenameTable
4676                                        || option.getOptionType() == EAlterTableOptionType.swapWith) {
4677                                TObjectName newTableName = option.getNewTableName();
4678                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
4679                                boolean containsTable = false;
4680                                for (int j = 0; j < list.size(); j++) {
4681                                        if (list.get(j) instanceof TTable) {
4682                                                TTable newTableTable = (TTable) list.get(j);
4683                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
4684                                                newNameTableModel.setStarStmt("rename_table");
4685
4686                                                TObjectName newStarColumn = new TObjectName();
4687                                                newStarColumn.setString("*");
4688                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
4689                                                                newStarColumn, true);
4690
4691                                                Process process = modelFactory.createProcess(stmt);
4692                                                newNameTableModel.addProcess(process);
4693                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4694                                                relation.setEffectType(
4695                                                                option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
4696                                                                                : EffectType.swap_table);                                               
4697                                                if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
4698                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4699                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4700                                                } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
4701                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4702                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4703                                                }
4704                                                relation.setProcess(process);
4705                                                containsTable = true;
4706
4707                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4708                                                        oldTableStarColumn.setShowStar(false);
4709                                                        relation.setShowStarRelation(false);
4710                                                }
4711
4712                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4713                                                        newTableStarColumn.setShowStar(false);
4714                                                        relation.setShowStarRelation(false);
4715                                                }
4716                                        }
4717                                }
4718                                if (!containsTable) {
4719                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
4720                                        newNameTableModel.setStarStmt("rename_table");
4721                                        TObjectName newStarColumn = new TObjectName();
4722                                        newStarColumn.setString("*");
4723                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
4724                                                        true);
4725
4726                                        Process process = modelFactory.createProcess(stmt);
4727                                        newNameTableModel.addProcess(process);
4728                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4729                                        relation.setEffectType(
4730                                                        option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
4731                                                                        : EffectType.swap_table);
4732                                        if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
4733                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4734                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4735                                        } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
4736                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4737                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4738                                        }
4739                                        relation.setProcess(process);
4740                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4741                                                oldTableStarColumn.setShowStar(false);
4742                                                relation.setShowStarRelation(false);
4743                                        }
4744
4745                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4746                                                newTableStarColumn.setShowStar(false);
4747                                                relation.setShowStarRelation(false);
4748                                        }
4749
4750                                }
4751                        }
4752                        else if (option.getOptionType() == EAlterTableOptionType.appendFrom) {
4753                                TObjectName newTableName = option.getSourceTableName();
4754                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
4755                                boolean containsTable = false;
4756                                for (int j = 0; j < list.size(); j++) {
4757                                        if (list.get(j) instanceof TTable) {
4758                                                TTable newTableTable = (TTable) list.get(j);
4759                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
4760                                                newNameTableModel.setStarStmt("append_from");
4761
4762                                                TObjectName newStarColumn = new TObjectName();
4763                                                newStarColumn.setString("*");
4764                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
4765                                                                newStarColumn, true);
4766
4767                                                Process process = modelFactory.createProcess(stmt);
4768                                                newNameTableModel.addProcess(process);
4769                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4770                                                relation.setEffectType(EffectType.append_from);
4771                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4772                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4773                                                relation.setProcess(process);
4774                                                containsTable = true;
4775
4776                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4777                                                        oldTableStarColumn.setShowStar(false);
4778                                                        relation.setShowStarRelation(false);
4779                                                }
4780
4781                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4782                                                        newTableStarColumn.setShowStar(false);
4783                                                        relation.setShowStarRelation(false);
4784                                                }
4785                                        }
4786                                }
4787                                if (!containsTable) {
4788                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
4789                                        newNameTableModel.setStarStmt("append_from");
4790                                        TObjectName newStarColumn = new TObjectName();
4791                                        newStarColumn.setString("*");
4792                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
4793                                                        true);
4794
4795                                        Process process = modelFactory.createProcess(stmt);
4796                                        newNameTableModel.addProcess(process);
4797                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4798                                        relation.setEffectType(EffectType.append_from);
4799                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4800                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4801                                        relation.setProcess(process);
4802                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4803                                                oldTableStarColumn.setShowStar(false);
4804                                                relation.setShowStarRelation(false);
4805                                        }
4806
4807                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4808                                                newTableStarColumn.setShowStar(false);
4809                                                relation.setShowStarRelation(false);
4810                                        }
4811
4812                                }
4813                        }
4814                        else if (option.getOptionType() == EAlterTableOptionType.exchangePartition) {
4815                                TObjectName newTableName = option.getNewTableName();
4816                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
4817                                boolean containsTable = false;
4818                                for (int j = 0; j < list.size(); j++) {
4819                                        if (list.get(j) instanceof TTable) {
4820                                                TTable newTableTable = (TTable) list.get(j);
4821                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
4822                                                newNameTableModel.setStarStmt("exchange_partition");
4823
4824                                                TObjectName newStarColumn = new TObjectName();
4825                                                newStarColumn.setString("*");
4826                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
4827                                                                newStarColumn, true);
4828
4829                                                Process process = modelFactory.createProcess(stmt);
4830                                                newNameTableModel.addProcess(process);
4831                                                {
4832                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4833                                                        relation.setEffectType(EffectType.exchange_partition);
4834                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4835                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4836                                                        relation.setProcess(process);
4837                                                        if (option.getPartitionName() != null) {
4838                                                                relation.setPartition(option.getPartitionName().toString());
4839                                                        }
4840                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4841                                                                oldTableStarColumn.setShowStar(false);
4842                                                                relation.setShowStarRelation(false);
4843                                                        }
4844
4845                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4846                                                                newTableStarColumn.setShowStar(false);
4847                                                                relation.setShowStarRelation(false);
4848                                                        }
4849                                                }
4850                                                {
4851                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4852                                                        relation.setEffectType(EffectType.exchange_partition);
4853                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4854                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4855                                                        relation.setProcess(process);
4856                                                        if (option.getPartitionName() != null) {
4857                                                                relation.setPartition(option.getPartitionName().toString());
4858                                                        }
4859                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4860                                                                oldTableStarColumn.setShowStar(false);
4861                                                                relation.setShowStarRelation(false);
4862                                                        }
4863
4864                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4865                                                                newTableStarColumn.setShowStar(false);
4866                                                                relation.setShowStarRelation(false);
4867                                                        }
4868                                                }
4869                                                containsTable = true;   
4870                                        }
4871                                }
4872                                if (!containsTable) {
4873                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
4874                                        newNameTableModel.setStarStmt("exchange_partition");
4875                                        TObjectName newStarColumn = new TObjectName();
4876                                        newStarColumn.setString("*");
4877                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
4878                                                        true);
4879
4880                                        Process process = modelFactory.createProcess(stmt);
4881                                        newNameTableModel.addProcess(process);
4882                                        {
4883                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4884                                                relation.setEffectType(EffectType.exchange_partition);
4885                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4886                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4887                                                if (option.getPartitionName() != null) {
4888                                                        relation.setPartition(option.getPartitionName().toString());
4889                                                }
4890                                                relation.setProcess(process);
4891                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4892                                                        oldTableStarColumn.setShowStar(false);
4893                                                        relation.setShowStarRelation(false);
4894                                                }
4895
4896                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4897                                                        newTableStarColumn.setShowStar(false);
4898                                                        relation.setShowStarRelation(false);
4899                                                }
4900                                        }
4901                                        {
4902                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4903                                                relation.setEffectType(EffectType.exchange_partition);
4904                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4905                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4906                                                if (option.getPartitionName() != null) {
4907                                                        relation.setPartition(option.getPartitionName().toString());
4908                                                }
4909                                                relation.setProcess(process);
4910                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4911                                                        oldTableStarColumn.setShowStar(false);
4912                                                        relation.setShowStarRelation(false);
4913                                                }
4914
4915                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4916                                                        newTableStarColumn.setShowStar(false);
4917                                                        relation.setShowStarRelation(false);
4918                                                }
4919                                        }
4920                                }
4921                        }
4922                        else if(option.getOptionType() == EAlterTableOptionType.setLocation) {
4923                                TObjectName location = option.getTableLocation();
4924                                Process process = modelFactory.createProcess(stmt);
4925                                process.setType("Set Table Location");
4926                                oldNameTableModel.addProcess(process);
4927                                Table uriFile = modelFactory.createTableByName(location, true);
4928                                uriFile.setPath(true);
4929                                for (int j = 0; j < oldNameTableModel.getColumns().size(); j++) {
4930                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4931                                        TObjectName fileUri = new TObjectName();
4932                                        fileUri.setString("*");
4933                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
4934                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4935                                        relation.setTarget(new TableColumnRelationshipElement(oldNameTableModel.getColumns().get(j)));
4936                                        relation.setProcess(process);
4937                                }
4938                        }
4939                        else if (option.getOptionType() == EAlterTableOptionType.AddColumn
4940                                        || option.getOptionType() == EAlterTableOptionType.addColumnIfNotExists) {
4941                                if (option.getColumnDefinitionList() != null) {
4942                                        for (TColumnDefinition column : option.getColumnDefinitionList()) {
4943                                                if (column != null && column.getColumnName() != null) {
4944                                                        TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column.getColumnName(), true);
4945                                                        if(this.option.getAnalyzeMode() == AnalyzeMode.crud) {
4946                                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
4947                                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
4948                                                                crudRelationship.setEffectType(EffectType.add_table_column);
4949                                                        }
4950                                                }
4951                                        }
4952                                }
4953                        }
4954                        else if (option.getOptionType() == EAlterTableOptionType.DropColumn && this.option.getAnalyzeMode() == AnalyzeMode.crud) {
4955                                if (option.getColumnNameList() != null) {
4956                                        for (TObjectName column : option.getColumnNameList()) {
4957                                                TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column, true);
4958                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
4959                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
4960                                                crudRelationship.setEffectType(EffectType.drop_table_column);
4961                                        }
4962                                }
4963                        }
4964                        else if(option.getOptionType() == EAlterTableOptionType.AddConstraint || option.getOptionType() == EAlterTableOptionType.AddConstraintFK
4965                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintPK || option.getOptionType() == EAlterTableOptionType.AddConstraintUnique
4966                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
4967                                if (option.getTableConstraint() != null) {
4968                                        TConstraint alertTableConstraint = option.getTableConstraint();
4969                                        TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
4970                                        for (int k = 0; k < keyNames.size(); k++) {
4971                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
4972                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
4973                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
4974                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
4975                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
4976                                                        tableConstraint.setPrimaryKey(true);
4977                                                }
4978                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
4979                                                        tableConstraint.setIndexKey(true);
4980                                                }
4981                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
4982                                                        tableConstraint.setUnqiueKey(true);
4983                                                }
4984                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
4985                                                        tableConstraint.setForeignKey(true);
4986                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
4987                                                        if (referencedTable == null) {
4988                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
4989                                                        }
4990                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
4991                                                        if (referencedTableColumns != null) {
4992                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
4993                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
4994                                                                                        referencedTableColumns.getObjectName(j), false);
4995                                                                        if (tableColumn != null) {
4996                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4997                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
4998                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
4999                                                                                relation.setEffectType(EffectType.foreign_key);
5000                                                                                Process process = modelFactory.createProcess(stmt);
5001                                                                                relation.setProcess(process);
5002                                                                                if(this.option.isShowERDiagram()){
5003                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5004                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5005                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5006                                                                                }
5007                                                                        }
5008                                                                }
5009                                                        }
5010                                                        else{
5011                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5012                                                                if (tableColumn != null) {
5013                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5014                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5015                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5016                                                                        relation.setEffectType(EffectType.foreign_key);
5017                                                                        if(this.option.isShowERDiagram()){
5018                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5019                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5020                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5021                                                                        }
5022                                                                }
5023                                                        }
5024                                                }
5025                                        }
5026                                }
5027                                else if (option.getConstraintList() != null) {
5028                                        for(int iCons=0; iCons<option.getConstraintList().size(); iCons++){
5029                                                TConstraint alertTableConstraint = option.getConstraintList().getConstraint(iCons);
5030                                                TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
5031                                                if(keyNames != null){
5032                                                        for (int k = 0; k < keyNames.size(); k++) {
5033                                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5034                                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
5035                                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5036                                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5037                                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
5038                                                                        tableConstraint.setPrimaryKey(true);
5039                                                                }
5040                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
5041                                                                        tableConstraint.setIndexKey(true);
5042                                                                }
5043                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
5044                                                                        tableConstraint.setUnqiueKey(true);
5045                                                                }
5046                                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
5047                                                                        tableConstraint.setForeignKey(true);
5048                                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5049                                                                        if (referencedTable == null) {
5050                                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5051                                                                        }
5052                                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
5053                                                                        if (referencedTableColumns != null) {
5054                                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5055                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5056                                                                                                        referencedTableColumns.getObjectName(j), false);
5057                                                                                        if (tableColumn != null) {
5058                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5059                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5060                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5061                                                                                                relation.setEffectType(EffectType.foreign_key);
5062                                                                                                Process process = modelFactory.createProcess(stmt);
5063                                                                                                relation.setProcess(process);
5064                                                                                                if(this.option.isShowERDiagram()){
5065                                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5066                                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5067                                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5068                                                                                                }
5069                                                                                        }
5070                                                                                }
5071                                                                        }
5072                                                                        else{
5073                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5074                                                                                if (tableColumn != null) {
5075                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5076                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5077                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5078                                                                                        relation.setEffectType(EffectType.foreign_key);
5079                                                                                        Process process = modelFactory.createProcess(stmt);
5080                                                                                        relation.setProcess(process);
5081                                                                                        if(this.option.isShowERDiagram()){
5082                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5083                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5084                                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5085                                                                                        }
5086                                                                                }
5087                                                                        }
5088                                                                }
5089                                                        }
5090                                                }
5091                                        }
5092                                }
5093                                else if (option.getIndexCols() != null){
5094                                        TPTNodeList<TColumnWithSortOrder> keyNames = option.getIndexCols();
5095                                        for (int k = 0; k < keyNames.size(); k++) {
5096                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5097                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5098                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5099                                                if(option.getOptionType() == EAlterTableOptionType.AddConstraintPK){
5100                                                        tableConstraint.setPrimaryKey(true);
5101                                                }
5102                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
5103                                                        tableConstraint.setIndexKey(true);
5104                                                }
5105                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintUnique){
5106                                                        tableConstraint.setUnqiueKey(true);
5107                                                }
5108                                                else if (option.getOptionType() == EAlterTableOptionType.AddConstraintFK) {
5109                                                        TObjectName referencedTableName = option.getReferencedObjectName();
5110                                                        tableConstraint.setForeignKey(true);
5111                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5112                                                        if (referencedTable == null) {
5113                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5114                                                        }
5115                                                        TObjectNameList referencedTableColumns = option.getReferencedColumnList();
5116                                                        if (referencedTableColumns != null) {
5117                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5118                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5119                                                                                        referencedTableColumns.getObjectName(j), false);
5120                                                                        if (tableColumn != null) {
5121                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5122                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5123                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5124                                                                                relation.setEffectType(EffectType.foreign_key);
5125                                                                                Process process = modelFactory.createProcess(stmt);
5126                                                                                relation.setProcess(process);
5127                                                                                if(this.option.isShowERDiagram()){
5128                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5129                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5130                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5131                                                                                }
5132                                                                        }
5133                                                                }
5134                                                        }
5135                                                        else{
5136                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5137                                                                if (tableColumn != null) {
5138                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5139                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5140                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5141                                                                        relation.setEffectType(EffectType.foreign_key);
5142                                                                        Process process = modelFactory.createProcess(stmt);
5143                                                                        relation.setProcess(process);
5144                                                                        if(this.option.isShowERDiagram()){
5145                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5146                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5147                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5148                                                                        }
5149                                                                }
5150                                                        }
5151                                                }
5152                                        }
5153                                }
5154                        }
5155                }
5156        }
5157
5158        private void analyzeAlterViewStmt(TAlterViewStatement stmt) {
5159                if (stmt.getAlterViewOption() == EAlterViewOption.asSelect) {
5160                        analyzeCreateViewStmt(stmt, stmt.getSelectSqlStatement(), null, stmt.getViewName());
5161                } else {
5162                        throw new UnsupportedOperationException("Can't handle this alter view statement case, alter option = "+ stmt.getAlterViewOption().name());
5163                }
5164        }
5165        
5166        private void analyzeDeleteStmt(TDeleteSqlStatement stmt) {
5167                TTable table = stmt.getTargetTable();
5168                if (table == null)
5169                        return;
5170                
5171                if (table.getCTE() != null) {
5172                        table = table.getCTE().getSubquery().getTables().getTable(0);
5173                } else if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) {
5174                        table = table.getLinkTable().getSubquery().getTables().getTable(0);
5175                } else if (table.getSubquery() != null) {
5176                        table = table.getSubquery().getTables().getTable(0);
5177                }
5178                
5179                Table tableModel = modelFactory.createTable(table);
5180                if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) {
5181                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
5182                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
5183
5184                                if (object.getDbObjectType() == EDbObjectType.variable) {
5185                                        continue;
5186                                }
5187
5188                                if (object.getColumnNameOnly().startsWith("@")
5189                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
5190                                        continue;
5191                                }
5192
5193                                if (object.getColumnNameOnly().startsWith(":")
5194                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
5195                                        continue;
5196                                }
5197
5198                                if (!isBuiltInFunctionName(object)) {
5199                                        if (object.getSourceTable() == null || object.getSourceTable() == table) {
5200                                                modelFactory.createTableColumn(tableModel, object, false);
5201                                        }
5202                                }
5203                        }
5204                }
5205
5206                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
5207                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5208                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
5209                        crudRelationship.setEffectType(EffectType.delete);
5210                }
5211                
5212                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
5213                        analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
5214                                        EffectType.delete);
5215                }
5216        }
5217
5218        private TObjectName getProcedureName(TStoredProcedureSqlStatement stmt) {
5219                if (stmt instanceof TTeradataCreateProcedure) {
5220                        return ((TTeradataCreateProcedure) stmt).getProcedureName();
5221                }
5222                return stmt.getStoredProcedureName();
5223        }
5224
5225        private void analyzePlsqlCreatePackage(TPlsqlCreatePackage stmt) {
5226                TObjectName procedureName = getProcedureName(stmt);
5227                OraclePackage oraclePackage;
5228                if (procedureName != null) {
5229                        if (this.modelManager.getOraclePackageByName(
5230                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))) == null) {
5231                                oraclePackage = this.modelFactory.createOraclePackage(stmt);
5232
5233                                if (stmt.getParameterDeclarations() != null) {
5234                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
5235
5236                                        for (int i = 0; i < parameters.size(); ++i) {
5237                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
5238                                                if (parameter.getParameterName() != null) {
5239                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
5240                                                } else if (parameter.getDataType() != null) {
5241                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
5242                                                }
5243                                        }
5244                                }
5245
5246                                ModelBindingManager.setGlobalOraclePackage(oraclePackage);
5247                        } else {
5248                                ModelBindingManager.setGlobalOraclePackage(this.modelManager.getOraclePackageByName(
5249                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))));
5250                        }
5251                        
5252                        try {
5253                                if (stmt.getDeclareStatements() != null) {
5254                                        for (int i = 0; i < stmt.getDeclareStatements().size(); ++i) {
5255                                                analyzeCustomSqlStmt(stmt.getDeclareStatements().get(i));
5256                                        }
5257                                }
5258                        } finally {
5259                                ModelBindingManager.removeGlobalOraclePackage();
5260                        }
5261                }
5262        }
5263
5264        private void analyzeStoredProcedureStmt(TStoredProcedureSqlStatement stmt) {
5265
5266                if (stmt instanceof TPlsqlCreatePackage) {
5267                        analyzePlsqlCreatePackage((TPlsqlCreatePackage) stmt);
5268                        return;
5269                }
5270                
5271                ModelBindingManager.setGlobalProcedure(stmt);
5272
5273                try {
5274                        Procedure procedure = null;
5275        
5276                        TObjectName procedureName = getProcedureName(stmt);
5277                        if (procedureName != null) {
5278                                procedure = this.modelFactory.createProcedure(stmt);
5279                                if (procedure != null) {
5280                                        modelManager.bindModel(stmt, procedure);
5281                                }
5282                                if (ModelBindingManager.getGlobalOraclePackage() != null) {
5283                                        ModelBindingManager.getGlobalOraclePackage().addProcedure(procedure);
5284                                        procedure.setParentPackage(ModelBindingManager.getGlobalOraclePackage());
5285                                }
5286                                if (stmt.getParameterDeclarations() != null) {
5287                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
5288        
5289                                        for (int i = 0; i < parameters.size(); ++i) {
5290                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
5291                                                Argument argument = null;
5292                                                TObjectName argumentName = null;
5293                                                if (parameter.getParameterName() != null) {
5294                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
5295                                                        argumentName = parameter.getParameterName();
5296                                                } else if (parameter.getDataType() != null) {
5297                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
5298                                                }
5299                                                
5300                                                if (argument != null) {
5301                                                        if (argumentName == null) {
5302                                                                argumentName = new TObjectName();
5303                                                                argumentName.setString(argument.getName());
5304                                                        }
5305                                                        Variable variable = modelFactory.createVariable(argument.getName());
5306                                                        if (argument.getMode() == EParameterMode.in || argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5307                                                                variable.setSubType(SubType.of(argument.getMode().name()));
5308                                                        } else {
5309                                                                variable.setSubType(SubType.argument);
5310                                                        }
5311                                                        if(isSimpleDataType(argument.getDataType())){
5312                                                                modelFactory.createTableColumn(variable, argumentName, true);
5313                                                        }
5314                                                        else {
5315                                                                TObjectName variableProperties = new TObjectName();
5316                                                                variableProperties.setString("*");
5317                                                                modelFactory.createTableColumn(variable, variableProperties, true);
5318                                                        }
5319                                                }
5320                                        }
5321                                }
5322                        }
5323        
5324                        if (stmt instanceof TCreateTriggerStmt) {
5325                                TCreateTriggerStmt trigger = (TCreateTriggerStmt) stmt;
5326        
5327                                if (trigger.getFunctionCall() != null) {
5328                                        modelFactory.createProcedureFromFunctionCall(trigger.getFunctionCall());
5329                                }
5330        
5331                                if (trigger.getTables() != null) {
5332                                        for (int i = 0; i < trigger.getTables().size(); i++) {
5333                                                Table tableModel = this.modelFactory.createTriggerOnTable(trigger.getTables().getTable(i));
5334                                        }
5335                                }
5336                        }
5337        
5338                        if (stmt instanceof TPlsqlCreateTrigger
5339                                        && ((TPlsqlCreateTrigger) stmt).getTriggeringClause().getEventClause() instanceof TDmlEventClause) {
5340                                TPlsqlCreateTrigger trigger = (TPlsqlCreateTrigger) stmt;
5341                                TDmlEventClause clause = (TDmlEventClause) ((TPlsqlCreateTrigger) stmt).getTriggeringClause()
5342                                                .getEventClause();
5343                                Table sourceTable = modelFactory.createTableByName(clause.getTableName());
5344        
5345                                for (TTriggerEventItem item : clause.getEventItems()) {
5346                                        if (item instanceof TDmlEventItem) {
5347                                                if (((TDmlEventItem) item).getColumnList() != null) {
5348                                                        for (TObjectName column : ((TDmlEventItem) item).getColumnList()) {
5349                                                                modelFactory.createTableColumn(sourceTable, column, true);
5350                                                        }
5351                                                }
5352                                        }
5353                                }
5354        
5355                                for (TCustomSqlStatement subStmt : ((TPlsqlCreateTrigger) stmt).getStatements()) {
5356                                        if (!(subStmt instanceof TCommonBlock))
5357                                                continue;
5358                                        for (TCustomSqlStatement blockSubStmt : ((TCommonBlock) subStmt).getStatements()) {
5359                                                if (!(blockSubStmt instanceof TBasicStmt))
5360                                                        continue;
5361                                                TBasicStmt basicStmt = (TBasicStmt) blockSubStmt;
5362                                                TExpression expression = basicStmt.getExpr();
5363                                                if (expression != null && expression.getExpressionType() == EExpressionType.function_t) {
5364                                                        Procedure targetProcedure = modelManager.getProcedureByName(DlineageUtil
5365                                                                        .getTableFullName(expression.getFunctionCall().getFunctionName().toString()));
5366                                                        if (targetProcedure == null) {
5367                                                                targetProcedure = modelManager
5368                                                                                .getProcedureByName(DlineageUtil.getTableFullName(procedure.getSchema() + "."
5369                                                                                                + expression.getFunctionCall().getFunctionName().toString()));
5370                                                        }
5371                                                        if (targetProcedure != null && expression.getFunctionCall().getArgs() != null && expression
5372                                                                        .getFunctionCall().getArgs().size() == targetProcedure.getArguments().size()) {
5373                                                                for (int j = 0; j < expression.getFunctionCall().getArgs().size(); j++) {
5374                                                                        TExpression columnExpr = expression.getFunctionCall().getArgs().getExpression(j);
5375                                                                        if (columnExpr.getExpressionType() == EExpressionType.simple_object_name_t) {
5376                                                                                TObjectName columnObject = columnExpr.getObjectOperand();
5377                                                                                if (columnObject.toString().indexOf(":") != -1) {
5378                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(sourceTable,
5379                                                                                                        columnObject, true);
5380                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5381                                                                                        relation.setEffectType(EffectType.trigger);
5382                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5383                                                                                        relation.setTarget(
5384                                                                                                        new ArgumentRelationshipElement(targetProcedure.getArguments().get(j)));
5385                                                                                        Process process = modelFactory.createProcess(stmt);
5386                                                                                        relation.setProcess(process);
5387                                                                                }
5388                                                                        }
5389                                                                }
5390                                                        }
5391                                                }
5392                                        }
5393                                }
5394                        }
5395        
5396                        if (stmt instanceof TMssqlCreateFunction) {
5397                                TMssqlCreateFunction createFunction = (TMssqlCreateFunction) stmt;
5398                                if (createFunction.getReturnTableVaraible() != null && createFunction.getReturnTableDefinitions() != null) {
5399                                        Table tableModel = this.modelFactory.createTableByName(createFunction.getReturnTableVaraible(), true);
5400                                        tableModel.setCreateTable(true);
5401                                        String procedureParent = createFunction.getFunctionName().toString();
5402                                        if (procedureParent != null) {
5403                                                tableModel.setParent(procedureParent);
5404                                        }
5405                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
5406        
5407                                        if (createFunction.getReturnTableDefinitions() != null) {
5408                                                for (int j = 0; j < createFunction.getReturnTableDefinitions().size(); j++) {
5409                                                        TTableElement tableElement = createFunction.getReturnTableDefinitions().getTableElement(j);
5410                                                        TColumnDefinition column = tableElement.getColumnDefinition();
5411                                                        if (column != null && column.getColumnName() != null) {
5412                                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
5413                                                        }
5414                                                }
5415                                        }
5416                                }
5417        
5418                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
5419                                        String procedureParent = createFunction.getFunctionName().toString();
5420                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
5421                                        ResultSet resultSetModel = (ResultSet) modelManager
5422                                                        .getModel(createFunction.getReturnStmt().getSubquery());
5423                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5424                                                        resultSetModel);
5425                                }
5426                        } else if (stmt instanceof TCreateFunctionStmt) {
5427                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
5428                                if (createFunction.getReturnDataType() != null
5429                                                && createFunction.getReturnDataType().getColumnDefList() != null) {
5430                                        Table tableModel = this.modelFactory.createTableByName(createFunction.getFunctionName(), true);
5431                                        tableModel.setCreateTable(true);
5432                                        String procedureParent = createFunction.getFunctionName().toString();
5433                                        if (procedureParent != null) {
5434                                                tableModel.setParent(procedureParent);
5435                                        }
5436        
5437                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
5438                                        for (int j = 0; j < createFunction.getReturnDataType().getColumnDefList().size(); j++) {
5439                                                TColumnDefinition column = createFunction.getReturnDataType().getColumnDefList().getColumn(j);
5440                                                if (column != null && column.getColumnName() != null) {
5441                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
5442                                                }
5443                                        }
5444        
5445                                        if (createFunction.getSqlQuery() != null) {
5446                                                analyzeSelectStmt(createFunction.getSqlQuery());
5447                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(createFunction.getSqlQuery());
5448                                                if (resultSetModel != null) {
5449                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
5450                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
5451                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
5452                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
5453                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(resultColumn.getName()),
5454                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
5455                                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5456                                                                                dataflowRelation.setEffectType(EffectType.select);
5457                                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
5458                                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
5459                                                                        }
5460                                                                }
5461                                                        }
5462                                                }
5463                                        }
5464                                }
5465                                
5466                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
5467                                        String procedureParent = createFunction.getFunctionName().toString();
5468                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
5469                                        ResultSet resultSetModel = (ResultSet) modelManager
5470                                                        .getModel(createFunction.getReturnStmt().getSubquery());
5471                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5472                                                        resultSetModel);
5473                                }
5474                                
5475                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getReturnExpr() != null) {
5476                                        TExpression returnExpression =  createFunction.getReturnStmt().getReturnExpr();
5477                                        ResultSet returnResult = modelFactory.createResultSet(createFunction.getReturnStmt(), false);
5478                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5479                                        
5480                                        TExpression expression = createFunction.getReturnStmt().getReturnExpr();
5481                                        analyzeResultColumnExpressionRelation(resultColumn, expression);
5482        
5483                                        String procedureParent = createFunction.getFunctionName().toString();
5484                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5485                                                        returnResult);
5486                                }
5487                        }
5488        
5489                        if (stmt instanceof TCreateProcedureStmt) {
5490                                TCreateProcedureStmt createProcedure = (TCreateProcedureStmt) stmt;
5491                                if (EDbVendor.dbvsnowflake == option.getVendor() && createProcedure.getRoutineBodyInConstant() != null) {
5492                                        extractSnowflakeSQLFromProcedure(createProcedure);
5493                                }
5494                        }
5495                                
5496                        if (stmt.getStatements().size() > 0) {
5497                                for (int i = 0; i < stmt.getStatements().size(); ++i) {
5498                                        this.analyzeCustomSqlStmt(stmt.getStatements().get(i));
5499                                }
5500                        }
5501
5502                        if (stmt.getBodyStatements().size() > 0) {
5503                                for (int i = 0; i < stmt.getBodyStatements().size(); ++i) {
5504                                        this.analyzeCustomSqlStmt(stmt.getBodyStatements().get(i));
5505                                }
5506                        }
5507                        
5508                        // Detect pipelined functions and build signatures
5509                        if (stmt instanceof TPlsqlCreateFunction && pipelinedAnalyzer != null) {
5510                                try {
5511                                        pipelinedAnalyzer.analyzePipelinedFunction((TPlsqlCreateFunction) stmt);
5512                                } catch (Exception e) {
5513                                        // Don't let pipelined analysis failure break main flow
5514                                }
5515                        }
5516
5517                        if (procedure != null && !getLastSelectStmt(stmt).isEmpty()) {
5518                                for (TSelectSqlStatement select : getLastSelectStmt(stmt)) {
5519                                        ResultSet resultSet = (ResultSet) modelManager.getModel(select);
5520                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedure.getName()),
5521                                                        resultSet);
5522                                }
5523//                              List<Argument> outArgs = new ArrayList<Argument>();
5524//                              for (int i = 0; i < procedure.getArguments().size(); i++) {
5525//                                      Argument argument = procedure.getArguments().get(i);
5526//                                      if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5527//                                              outArgs.add(argument);
5528//                                      }
5529//                              }
5530//
5531//                              if (resultSet != null && resultSet.getColumns().size() == outArgs.size()) {
5532//                                      for (int i = 0; i < outArgs.size(); i++) {
5533//                                              Argument argument = outArgs.get(i);
5534//                                              Variable variable = modelFactory.createVariable(argument.getName(), false);
5535//                                              if (variable != null) {
5536//                                                      DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5537//                                                      dataflowRelation.setEffectType(EffectType.output);
5538//                                                      dataflowRelation
5539//                                                                      .addSource(new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
5540//                                                      dataflowRelation
5541//                                                                      .setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0)));
5542//                                              }
5543//                                      }
5544//
5545//                              }
5546                        }
5547
5548                        if (stmt instanceof TCreateFunctionStmt && ((TCreateFunctionStmt)stmt).getSqlExpression()!=null) {
5549                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
5550                                TExpression returnExpression = createFunction.getSqlExpression();
5551
5552                                ResultSet returnResult = modelFactory.createResultSet(stmt, false);
5553                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5554
5555                                columnsInExpr visitor = new columnsInExpr();
5556                                returnExpression.inOrderTraverse(visitor);
5557
5558                                List<TObjectName> objectNames = visitor.getObjectNames();
5559                                List<TParseTreeNode> functions = visitor.getFunctions();
5560                                List<TParseTreeNode> constants = visitor.getConstants();
5561                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5562
5563                                if (functions != null && !functions.isEmpty()) {
5564                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5565                                }
5566                                if (subquerys != null && !subquerys.isEmpty()) {
5567                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5568                                }
5569                                if (objectNames != null && !objectNames.isEmpty()) {
5570                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5571                                }
5572                                if (constants != null && !constants.isEmpty()) {
5573                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5574                                }
5575
5576                                String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5577                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5578                                                returnResult);
5579                        }
5580                } finally {
5581                        ModelBindingManager.removeGlobalProcedure();
5582                }
5583
5584        }
5585
5586        private boolean isSimpleDataType(TTypeName dataType) {
5587                if (dataType.getDataType() == EDataType.variant_t) {
5588                        return false;
5589                }
5590                if (dataType.getDataType() == EDataType.cursor_t) {
5591                        return false;
5592                }
5593                if (dataType.getDataType() == EDataType.generic_t) {
5594                        return false;
5595                }
5596                if (dataType.getDataType() == EDataType.unknown_t) {
5597                        return false;
5598                }
5599                if (dataType.getDataType() == EDataType.sql_variant_t) {
5600                        return false;
5601                }
5602                if (dataType.getDataType() == EDataType.table_t) {
5603                        return false;
5604                }
5605                if (dataType.getDataType() == EDataType.raw_t) {
5606                        return false;
5607                }
5608                if (dataType.getDataType() == EDataType.resultset_t) {
5609                        return false;
5610                }
5611                if (dataType.getDataType() == EDataType.row_t) {
5612                        return false;
5613                }
5614                if (dataType.getDataType() == EDataType.map_t) {
5615                        return false;
5616                }
5617                if (dataType.getDataType() == EDataType.anyType_t) {
5618                        return false;
5619                }
5620                if (dataType.getDataType() == EDataType.struct_t) {
5621                        return false;
5622                }
5623                if (dataType.getDataType() == EDataType.structType_t) {
5624                        return false;
5625                }
5626                if (dataType.getDataType() == EDataType.mapType_t) {
5627                        return false;
5628                }
5629                return true;
5630        }
5631
5632        protected void analyzeResultColumnExpressionRelation(Object resultColumn, TExpression expression) {
5633                columnsInExpr visitor = new columnsInExpr();
5634                expression.inOrderTraverse(visitor);
5635                List<TObjectName> objectNames = visitor.getObjectNames();
5636                List<TParseTreeNode> functions = visitor.getFunctions();
5637                List<TParseTreeNode> constants = visitor.getConstants();
5638                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5639
5640                if (functions != null && !functions.isEmpty()) {
5641                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5642                }
5643                if (subquerys != null && !subquerys.isEmpty()) {
5644                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5645                }
5646                if (objectNames != null && !objectNames.isEmpty()) {
5647                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5648                }
5649                if (constants != null && !constants.isEmpty()) {
5650                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5651                }
5652        }
5653
5654        private void analyzeDb2ReturnStmt(TDb2ReturnStmt stmt) {
5655                if (stmt.getReturnExpr() != null) {
5656                        TExpression returnExpression = stmt.getReturnExpr();
5657                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5658                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5659                        
5660                        columnsInExpr visitor = new columnsInExpr();
5661                        stmt.getReturnExpr().inOrderTraverse(visitor);
5662                        
5663                        List<TObjectName> objectNames = visitor.getObjectNames();
5664                        List<TParseTreeNode> functions = visitor.getFunctions();
5665                        List<TParseTreeNode> constants = visitor.getConstants();
5666                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5667
5668                        if (functions != null && !functions.isEmpty()) {
5669                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5670                        }
5671                        if (subquerys != null && !subquerys.isEmpty()) {
5672                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5673                        }
5674                        if (objectNames != null && !objectNames.isEmpty()) {
5675                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5676                        }
5677                        if (constants != null && !constants.isEmpty()) {
5678                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5679                        }
5680
5681                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5682                        if (procedureParent != null) {
5683                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5684                                                returnResult);
5685                        }
5686                }
5687        }
5688
5689        private void analyzeReturnStmt(TReturnStmt stmt) {
5690                if (stmt.getResultColumnList() != null) {
5691                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5692                        for (TResultColumn column : stmt.getResultColumnList()) {
5693                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
5694
5695                                columnsInExpr visitor = new columnsInExpr();
5696                                column.getExpr().inOrderTraverse(visitor);
5697
5698                                List<TObjectName> objectNames = visitor.getObjectNames();
5699                                List<TParseTreeNode> functions = visitor.getFunctions();
5700                                List<TParseTreeNode> constants = visitor.getConstants();
5701                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5702
5703                                if (functions != null && !functions.isEmpty()) {
5704                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5705                                }
5706                                if (subquerys != null && !subquerys.isEmpty()) {
5707                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5708                                }
5709                                if (objectNames != null && !objectNames.isEmpty()) {
5710                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5711                                }
5712                                if (constants != null && !constants.isEmpty()) {
5713                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5714                                }
5715
5716                        }
5717
5718                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5719                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5720                                        returnResult);
5721                }
5722                else if(stmt.getExpression()!=null){
5723                        TExpression returnExpression = stmt.getExpression();
5724                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5725                        
5726                        columnsInExpr visitor = null;
5727                        List<TSelectSqlStatement> subquerys = null;
5728                        
5729                        if (returnExpression.getFunctionCall() != null
5730                                        && returnExpression.getFunctionCall().getFunctionName().toString().equalsIgnoreCase("table")
5731                                        && returnExpression.getFunctionCall().getArgs() != null
5732                                        && returnExpression.getFunctionCall().getArgs().size()>0) {
5733                                visitor = new columnsInExpr();
5734                                returnExpression.getFunctionCall().getArgs().getExpression(0).inOrderTraverse(visitor);
5735                                subquerys = visitor.getSubquerys();
5736                                if (subquerys != null && !subquerys.isEmpty()) {
5737                                        analyzeSelectStmt(subquerys.get(0));
5738                                        ResultSet resultSet = (ResultSet) modelManager.getModel(subquerys.get(0));
5739                                        if (resultSet != null && resultSet.getColumns() != null) {
5740                                                for (ResultColumn column : resultSet.getColumns()) {
5741                                                        TObjectName columnName = new TObjectName();
5742                                                        columnName.setString(column.getName());
5743                                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, columnName);
5744                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5745                                                        dataflowRelation.setEffectType(EffectType.select);
5746                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
5747                                                        dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
5748                                                }
5749                                        }
5750                                        returnResult.setDetermined(resultSet.isDetermined());
5751                                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5752                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5753                                                        returnResult);
5754                                        return;
5755                                }
5756                        }
5757                        
5758                        ResultColumn resultColumn = null;
5759                        if (returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
5760                                TObjectName columnName = new TObjectName();
5761                                columnName.setString("*");
5762                                resultColumn = modelFactory.createResultColumn(returnResult, columnName);
5763                        }
5764                        else {
5765                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5766                        }
5767
5768                        visitor = new columnsInExpr();
5769                        stmt.getExpression().inOrderTraverse(visitor);
5770
5771                        List<TObjectName> objectNames = visitor.getObjectNames();
5772                        List<TParseTreeNode> functions = visitor.getFunctions();
5773                        List<TParseTreeNode> constants = visitor.getConstants();
5774                        subquerys = visitor.getSubquerys();
5775
5776                        if (functions != null && !functions.isEmpty()) {
5777                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5778                        }
5779                        if (subquerys != null && !subquerys.isEmpty()) {
5780                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5781                        }
5782                        if (objectNames != null && !objectNames.isEmpty()) {
5783                                DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5784                                //如果variable对应的不是一个复杂结构，则resultColumn不要设置为*
5785                                if (relation != null && relation.getTarget().getElement() == resultColumn && relation.getSources().size() == 1) {
5786                                        Object column = relation.getSources().iterator().next().getElement();
5787                                        boolean star = true;
5788                                        if (column instanceof TableColumn && ((TableColumn) column).getName().indexOf("*") == -1) {
5789                                                star = false;
5790                                        }
5791                                        if (column instanceof ResultColumn && ((ResultColumn) column).getName().indexOf("*") == -1) {
5792                                                star = false;
5793                                        }
5794                                        if (!star && returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
5795                                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression.getObjectOperand());
5796                                                returnResult.getColumns().clear();
5797                                                returnResult.addColumn(resultColumn);
5798                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
5799                                        }
5800                                }
5801                        }
5802                        if (constants != null && !constants.isEmpty()) {
5803                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5804                        }
5805
5806                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5807                        if (procedureParent != null) {
5808                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5809                                                returnResult);
5810                        }
5811                }
5812        }
5813
5814        private void analyzeMssqlReturnStmt(TMssqlReturn stmt) {
5815                if (stmt.getResultColumnList() != null) {
5816                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5817                        for (TResultColumn column : stmt.getResultColumnList()) {
5818                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
5819
5820                                columnsInExpr visitor = new columnsInExpr();
5821                                column.getExpr().inOrderTraverse(visitor);
5822
5823                                List<TObjectName> objectNames = visitor.getObjectNames();
5824                                List<TParseTreeNode> functions = visitor.getFunctions();
5825                                List<TParseTreeNode> constants = visitor.getConstants();
5826                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5827
5828                                if (functions != null && !functions.isEmpty()) {
5829                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5830                                }
5831                                if (subquerys != null && !subquerys.isEmpty()) {
5832                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5833                                }
5834                                if (objectNames != null && !objectNames.isEmpty()) {
5835                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5836                                }
5837                                if (constants != null && !constants.isEmpty()) {
5838                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5839                                }
5840
5841                        }
5842
5843                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5844                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5845                                        returnResult);
5846                }
5847                else if(stmt.getReturnExpr()!=null){
5848                        TExpression returnExpression = stmt.getReturnExpr();
5849                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5850                        
5851                        if (returnExpression.getSubQuery() == null) {
5852                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5853                                columnsInExpr visitor = new columnsInExpr();
5854                                stmt.getReturnExpr().inOrderTraverse(visitor);
5855
5856                                List<TObjectName> objectNames = visitor.getObjectNames();
5857                                List<TParseTreeNode> functions = visitor.getFunctions();
5858                                List<TParseTreeNode> constants = visitor.getConstants();
5859                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5860
5861                                if (functions != null && !functions.isEmpty()) {
5862                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5863                                }
5864                                if (subquerys != null && !subquerys.isEmpty()) {
5865                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5866                                }
5867                                if (objectNames != null && !objectNames.isEmpty()) {
5868                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5869                                }
5870                                if (constants != null && !constants.isEmpty()) {
5871                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5872                                }
5873                        }
5874                        else {
5875                                analyzeSelectStmt(returnExpression.getSubQuery());
5876                                ResultSet subResultSet = (ResultSet)modelManager.getModel(returnExpression.getSubQuery());
5877                                if (subResultSet != null) {
5878                                        for (ResultColumn subResultColumn : subResultSet.getColumns()) {
5879                                                TObjectName objectName = new TObjectName();
5880                                                objectName.setString(subResultColumn.getName());
5881                                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, objectName);
5882                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5883                                                dataflowRelation.setEffectType(EffectType.select);
5884                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(subResultColumn));
5885                                                dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
5886
5887                                        }
5888                                        
5889                                        if(subResultSet.getRelationRows().hasRelation()) {
5890                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
5891                                                impactRelation.setEffectType(EffectType.select);
5892                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
5893                                                                subResultSet.getRelationRows()));
5894                                                impactRelation.setTarget(
5895                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(returnResult.getRelationRows()));
5896                                        }
5897                                }
5898                        }
5899                        
5900                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5901                        if (procedureParent != null) {
5902                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5903                                                returnResult);
5904                        }
5905                }
5906        }
5907        
5908        private void analyzeFetchStmt(TFetchStmt stmt) {
5909                if (stmt.getVariableNames() != null) {
5910                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
5911                                TExpression variableExpression = stmt.getVariableNames().getExpression(i);
5912                                if (variableExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
5913                                        TObjectName columnObject = variableExpression.getObjectOperand();
5914                                        if (columnObject.getDbObjectType() == EDbObjectType.variable) {
5915                                                continue;
5916                                        }
5917
5918                                        if (columnObject.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
5919                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
5920                                                continue;
5921                                        }
5922
5923                                        if (columnObject.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
5924                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
5925                                                continue;
5926                                        }
5927
5928                                        Variable cursorVariable = modelFactory.createVariable(columnObject);
5929                                        cursorVariable.setSubType(SubType.record);
5930                                        if (cursorVariable.isDetermined()) {
5931                                                if (stmt.getCursorName() != null) {
5932                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
5933                                                        String variableString = stmt.getCursorName().toString();
5934                                                        if (variableString.startsWith(":")) {
5935                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
5936                                                        }
5937                                                        if (!SQLUtil.isEmpty(procedureName)) {
5938                                                                variableString = procedureName + "."
5939                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
5940                                                        }
5941                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
5942                                                        if (cursor != null) {
5943                                                                for (TableColumn variableProperty : cursorVariable.getColumns()) {
5944                                                                        boolean flag = false;
5945                                                                        for (TableColumn cursorColumn : cursor.getColumns()) {
5946                                                                                if (getColumnName(cursorColumn.getName())
5947                                                                                                .equalsIgnoreCase(variableProperty.getName())) {
5948                                                                                        DataFlowRelationship dataflowRelation = modelFactory
5949                                                                                                        .createDataFlowRelation();
5950                                                                                        dataflowRelation.setEffectType(EffectType.cursor);
5951                                                                                        dataflowRelation
5952                                                                                                        .addSource(new TableColumnRelationshipElement(cursorColumn));
5953                                                                                        dataflowRelation
5954                                                                                                        .setTarget(new TableColumnRelationshipElement(variableProperty));
5955                                                                                        flag = true;
5956                                                                                        break;
5957                                                                                }
5958                                                                        }
5959
5960                                                                        if (!flag) {
5961                                                                                if (cursor.getColumns().size() == stmt.getVariableNames().size()) {
5962                                                                                        DataFlowRelationship dataflowRelation = modelFactory
5963                                                                                                        .createDataFlowRelation();
5964                                                                                        dataflowRelation.setEffectType(EffectType.cursor);
5965                                                                                        dataflowRelation
5966                                                                                                        .setTarget(new TableColumnRelationshipElement(variableProperty));
5967                                                                                        dataflowRelation.addSource(
5968                                                                                                        new TableColumnRelationshipElement(cursor.getColumns().get(i)));
5969                                                                                }
5970                                                                                else {
5971                                                                                        for (int j = 0; j < cursor.getColumns().size(); j++) {
5972                                                                                                DataFlowRelationship dataflowRelation = modelFactory
5973                                                                                                                .createDataFlowRelation();
5974                                                                                                dataflowRelation.setEffectType(EffectType.cursor);
5975                                                                                                if (stmt.getVariableNames().size() == 1) {
5976                                                                                                        dataflowRelation.addSource(new TableColumnRelationshipElement(
5977                                                                                                                        cursor.getColumns().get(j)));
5978                                                                                                } else {
5979                                                                                                        dataflowRelation.addSource(new TableColumnRelationshipElement(
5980                                                                                                                        cursor.getColumns().get(j), i));
5981                                                                                                }
5982                                                                                                dataflowRelation.setTarget(
5983                                                                                                                new TableColumnRelationshipElement(variableProperty));
5984                                                                                        }
5985                                                                                }
5986                                                                        }
5987                                                                }
5988                                                        }
5989                                                }
5990                                                
5991                                        } else {
5992                                                TableColumn variableProperty = null;
5993                                                if (stmt.getVariableNames().size() == 1) {
5994                                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
5995                                                                TObjectName starColumn = new TObjectName();
5996                                                                starColumn.setString("*");
5997                                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
5998                                                        } else {
5999                                                                variableProperty = cursorVariable.getColumns().get(0);
6000                                                        }
6001                                                } else {
6002                                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
6003                                                }
6004
6005                                                if (stmt.getCursorName() != null) {
6006                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6007                                                        String variableString = stmt.getCursorName().toString();
6008                                                        if (variableString.startsWith(":")) {
6009                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6010                                                        }
6011                                                        if (!SQLUtil.isEmpty(procedureName)) {
6012                                                                variableString = procedureName + "."
6013                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
6014                                                        }
6015                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
6016                                                        if (cursor != null) {
6017                                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
6018                                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6019                                                                        dataflowRelation.setEffectType(EffectType.cursor);
6020                                                                        if (stmt.getVariableNames().size() == 1) {
6021                                                                                dataflowRelation.addSource(
6022                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j)));
6023                                                                        } else {
6024                                                                                dataflowRelation.addSource(
6025                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
6026                                                                        }
6027                                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6028                                                                }
6029                                                        }
6030                                                }
6031                                        }
6032                                }
6033                        }
6034                }
6035        }
6036
6037        private void analyzeFetchStmt(TMssqlFetch stmt) {
6038                if (stmt.getVariableNames() != null) {
6039                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
6040                                TObjectName columnObject = stmt.getVariableNames().getObjectName(i);
6041                                Variable cursorVariable = modelFactory.createVariable(columnObject);
6042                                cursorVariable.setCreateTable(true);
6043                                cursorVariable.setSubType(SubType.record);
6044                                TableColumn variableProperty = null;
6045                                if (stmt.getVariableNames().size() == 1) {
6046                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
6047                                                TObjectName starColumn = new TObjectName();
6048                                                starColumn.setString("*");
6049                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6050                                        } else {
6051                                                variableProperty = cursorVariable.getColumns().get(0);
6052                                        }
6053                                } else {
6054                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
6055                                }
6056
6057                                if (stmt.getCursorName() != null) {
6058                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6059                                        String variableString = stmt.getCursorName().toString();
6060                                        if (variableString.startsWith(":")) {
6061                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6062                                        }
6063                                        if (!SQLUtil.isEmpty(procedureName)) {
6064                                                variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
6065                                        }
6066                                        Table cursor = modelManager
6067                                                        .getTableByName(DlineageUtil.getTableFullName(variableString));
6068                                        if (cursor != null) {
6069                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
6070                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6071                                                        dataflowRelation.setEffectType(EffectType.cursor);
6072                                                        if (stmt.getVariableNames().size() == 1) {
6073                                                                dataflowRelation
6074                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
6075                                                        } else {
6076                                                                dataflowRelation
6077                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
6078                                                        }
6079                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6080                                                }
6081                                        }
6082                                }
6083                        }
6084
6085                }
6086        }
6087
6088        private void analyzeLoopStmt(TLoopStmt stmt) {
6089
6090                if (stmt.getCursorName() != null && stmt.getIndexName() != null) {
6091                        modelManager.bindCursorIndex(stmt.getIndexName(), stmt.getCursorName());
6092                }
6093
6094                if (stmt.getRecordName() != null && stmt.getSubquery() != null) {
6095                        Variable cursorTempTable = modelFactory.createCursor(stmt);
6096                        cursorTempTable.setVariable(true);
6097                        cursorTempTable.setSubType(SubType.cursor);
6098                        modelManager.bindCursorModel(stmt, cursorTempTable);
6099                        analyzeSelectStmt(stmt.getSubquery());
6100
6101                        TableColumn cursorColumn = null;
6102                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6103                                TObjectName starColumn = new TObjectName();
6104                                starColumn.setString("*");
6105                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6106                        } else {
6107                                cursorColumn = cursorTempTable.getColumns().get(0);
6108                        }
6109
6110
6111                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6112                        if (resultSetModel != null) {
6113                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6114                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6115                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6116                                        dataflowRelation.setEffectType(EffectType.cursor);
6117                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6118                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6119                                }
6120                        }
6121                }
6122
6123                for (int i = 0; i < stmt.getStatements().size(); i++) {
6124                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
6125                }
6126        }
6127
6128        private void analyzeForStmt(TForStmt stmt) {
6129                if (stmt.getSubquery() == null) {
6130                        return;
6131                }
6132
6133                Variable cursorTempTable = modelFactory.createCursor(stmt);
6134                cursorTempTable.setVariable(true);
6135                cursorTempTable.setSubType(SubType.cursor);
6136                cursorTempTable.setCreateTable(true);
6137                modelManager.bindCursorModel(stmt, cursorTempTable);
6138                analyzeSelectStmt(stmt.getSubquery());
6139
6140                TableColumn cursorColumn = null;
6141                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6142                        TObjectName starColumn = new TObjectName();
6143                        starColumn.setString("*");
6144                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6145                } else {
6146                        cursorColumn = cursorTempTable.getColumns().get(0);
6147                }
6148
6149
6150                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6151                if (resultSetModel != null) {
6152                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6153                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6154                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6155                                dataflowRelation.setEffectType(EffectType.cursor);
6156                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6157                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6158                        }
6159                }
6160
6161                if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
6162                        for (int i = 0; i < stmt.getStatements().size(); i++) {
6163                                analyzeCustomSqlStmt(stmt.getStatements().get(i));
6164                        }
6165                }
6166        }
6167
6168        private void analyzeVarDeclStmt(TVarDeclStmt stmt) {
6169                TTypeName typeName = stmt.getDataType();
6170                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
6171                        Variable cursorVariable = modelFactory.createVariable(stmt.getElementName());
6172                        cursorVariable.setSubType(SubType.record_type);
6173
6174                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
6175                        if(!variableTable.isCreateTable()) {
6176                                TObjectName starColumn1 = new TObjectName();
6177                                starColumn1.setString("*");
6178                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
6179                                variableTableStarColumn.setShowStar(false);
6180                                variableTableStarColumn.setExpandStar(true);
6181
6182                                TObjectName starColumn = new TObjectName();
6183                                starColumn.setString("*");
6184                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6185                                variableProperty.setShowStar(false);
6186                                variableProperty.setExpandStar(true);
6187
6188                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6189                                dataflowRelation.setEffectType(EffectType.rowtype);
6190                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
6191                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6192                        } else {
6193                                for (TableColumn sourceColumn : variableTable.getColumns()) {
6194                                        String columnName = sourceColumn.getName();
6195                                        TObjectName targetColumn = new TObjectName();
6196                                        targetColumn.setString(columnName);
6197                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
6198                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6199                                        dataflowRelation.setEffectType(EffectType.rowtype);
6200                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
6201                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6202                                }
6203                        }
6204                } else if (stmt.getElementName() != null) {
6205                        Variable variable = modelFactory.createVariable(stmt.getElementName());
6206                        variable.setCreateTable(true);
6207                        variable.setSubType(SubType.record);
6208                        TableColumn tableColumn = null;
6209                        if (stmt.getDataType() != null && (modelFactory.createVariable(stmt.getDataType().getDataTypeName(), false)!=null || isCursorType(stmt.getDataType().getDataTypeName()))) {
6210                                Variable cursorVariable = modelFactory.createVariable(stmt.getDataType().getDataTypeName(), false);
6211                                if (cursorVariable != null) {
6212                                        if (cursorVariable.getSubType() == SubType.record_type) {
6213                                                variable.setSubType(SubType.record_type);
6214                                        }
6215                                        if(cursorVariable.isDetermined()) {
6216                                                for (int k = 0; k < cursorVariable.getColumns().size(); k++) {
6217                                                        TableColumn sourceColumn = cursorVariable.getColumns().get(k);
6218                                                        TObjectName objectName = new TObjectName();
6219                                                        objectName.setString(sourceColumn.getName());
6220                                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);
6221                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6222                                                        dataflowRelation.setEffectType(EffectType.cursor);
6223                                                        dataflowRelation
6224                                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
6225                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
6226                                                }
6227                                                variable.setDetermined(true);
6228                                                return;
6229                                        }
6230                                        else {
6231                                                TObjectName objectName = new TObjectName();
6232                                                objectName.setString("*");
6233                                                tableColumn = modelFactory.createTableColumn(variable, objectName, true);
6234                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6235                                                dataflowRelation.setEffectType(EffectType.cursor);
6236                                                dataflowRelation
6237                                                                .addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
6238                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
6239                                        }
6240                                }
6241                                else {
6242                                        TObjectName objectName = new TObjectName();
6243                                        objectName.setString("*");
6244                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);                                       
6245                                }
6246                        } else {
6247                                tableColumn = modelFactory.createTableColumn(variable, stmt.getElementName(), true);
6248                                tableColumn.setVariant(true);
6249                        }
6250
6251                        if (stmt.getDefaultValue() != null) {
6252                                columnsInExpr visitor = new columnsInExpr();
6253                                stmt.getDefaultValue().inOrderTraverse(visitor);
6254                                List<TObjectName> objectNames = visitor.getObjectNames();
6255                                List<TParseTreeNode> functions = visitor.getFunctions();
6256                                List<TParseTreeNode> constants = visitor.getConstants();
6257                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6258
6259                                if (functions != null && !functions.isEmpty()) {
6260                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6261                                }
6262                                if (subquerys != null && !subquerys.isEmpty()) {
6263                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6264                                }
6265                                if (objectNames != null && !objectNames.isEmpty()) {
6266                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6267                                }
6268                                if (constants != null && !constants.isEmpty()) {
6269                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6270                                }
6271                        }
6272                        
6273                        
6274                }
6275        }
6276
6277        private boolean isCursorType(String dataTypeName) {
6278                if (dataTypeName != null && dataTypeName.toLowerCase().contains("cursor")) {
6279                        return true;
6280                }
6281                return false;
6282        }
6283
6284        private void analyzeSetStmt(TSetStmt stmt) {
6285                TExpression right = stmt.getVariableValue();
6286                TObjectName columnObject = stmt.getVariableName();
6287                if (columnObject != null) {
6288                        TableColumn tableColumn = null;
6289                        Variable tableModel;
6290                        if (columnObject.toString().indexOf(".") != -1) {
6291                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6292                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6293                        } else {
6294                                tableModel = modelFactory.createVariable(columnObject);
6295                        }
6296                        tableModel.setCreateTable(true);
6297                        tableModel.setSubType(SubType.record);
6298
6299                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6300                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6301                        } else {
6302                                tableColumn = tableModel.getColumns().get(0);
6303                        }
6304
6305                        if (tableColumn != null && right!=null) {
6306                                columnsInExpr visitor = new columnsInExpr();
6307                                right.inOrderTraverse(visitor);
6308                                List<TObjectName> objectNames = visitor.getObjectNames();
6309                                List<TParseTreeNode> functions = visitor.getFunctions();
6310                                List<TParseTreeNode> constants = visitor.getConstants();
6311                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6312
6313                                if (functions != null && !functions.isEmpty()) {
6314                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6315                                }
6316                                if (subquerys != null && !subquerys.isEmpty()) {
6317                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6318                                }
6319                                if (objectNames != null && !objectNames.isEmpty()) {
6320                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6321                                }
6322                                if (constants != null && !constants.isEmpty()) {
6323                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6324                                }
6325                                
6326                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6327                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6328                                }
6329                        }
6330                }
6331
6332                if(stmt.getAssignments()!=null){
6333                        for (int i = 0; i < stmt.getAssignments().size(); i++) {
6334                                TSetAssignment assignStmt = stmt.getAssignments().getElement(i);
6335                                analyzeSetAssignmentStmt(assignStmt);
6336                        }
6337                }
6338        }
6339
6340        private void analyzeSetAssignmentStmt(TSetAssignment stmt) {
6341                TExpression right = stmt.getParameterValue();
6342                TObjectName columnObject = stmt.getParameterName();
6343                if (columnObject != null) {
6344                        TableColumn tableColumn = null;
6345                        Variable tableModel;
6346                        if (columnObject.toString().indexOf(".") != -1) {
6347                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6348                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6349                        } else {
6350                                tableModel = modelFactory.createVariable(columnObject);
6351                        }
6352                        tableModel.setCreateTable(true);
6353                        tableModel.setSubType(SubType.record);
6354                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6355                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6356                        } else {
6357                                tableColumn = tableModel.getColumns().get(0);
6358                        }
6359
6360                        if (tableColumn != null && right!=null) {
6361                                columnsInExpr visitor = new columnsInExpr();
6362                                right.inOrderTraverse(visitor);
6363                                List<TObjectName> objectNames = visitor.getObjectNames();
6364                                List<TParseTreeNode> functions = visitor.getFunctions();
6365                                List<TParseTreeNode> constants = visitor.getConstants();
6366                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6367
6368                                if (functions != null && !functions.isEmpty()) {
6369                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6370                                }
6371                                if (subquerys != null && !subquerys.isEmpty()) {
6372                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6373                                }
6374                                if (objectNames != null && !objectNames.isEmpty()) {
6375                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6376                                }
6377                                if (constants != null && !constants.isEmpty()) {
6378                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6379                                }
6380
6381                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6382                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6383                                }
6384                        }
6385                }
6386        }
6387
6388        private void analyzeMssqlSetStmt(TMssqlSet stmt) {
6389                TExpression right = stmt.getVarExpr();
6390                TObjectName columnObject = stmt.getVarName();
6391                if (columnObject != null) {
6392                        TableColumn tableColumn = null;
6393                        Variable tableModel;
6394                        if (columnObject.toString().indexOf(".") != -1) {
6395                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6396                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6397                        } else {
6398                                tableModel = modelFactory.createVariable(columnObject);
6399                        }
6400                        tableModel.setCreateTable(true);
6401                        tableModel.setSubType(SubType.record);
6402                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6403                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6404                        }
6405                        else {
6406                                tableColumn = tableModel.getColumns().get(0);
6407                        }
6408
6409                        if (tableColumn != null && right!=null) {
6410                                columnsInExpr visitor = new columnsInExpr();
6411                                right.inOrderTraverse(visitor);
6412                                List<TObjectName> objectNames = visitor.getObjectNames();
6413                                List<TParseTreeNode> functions = visitor.getFunctions();
6414                                List<TParseTreeNode> constants = visitor.getConstants();
6415                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6416
6417                                if (functions != null && !functions.isEmpty()) {
6418                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6419                                }
6420                                if (subquerys != null && !subquerys.isEmpty()) {
6421                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6422                                }
6423                                if (objectNames != null && !objectNames.isEmpty()) {
6424                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6425                                }
6426                                if (constants != null && !constants.isEmpty()) {
6427                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6428                                }
6429
6430                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6431                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6432                                }
6433                        }
6434                        else if(tableColumn!=null && stmt.getSubquery()!=null){
6435                                analyzeCustomSqlStmt(stmt.getSubquery());
6436                                analyzeSubqueryDataFlowRelation(tableColumn, Arrays.asList(stmt.getSubquery()), EffectType.select);
6437                        }
6438                }
6439        }
6440
6441        private void analyzeAssignStmt(TAssignStmt stmt) {
6442                TExpression left = stmt.getLeft();
6443                TExpression right = stmt.getExpression();
6444                TObjectName columnObject = null;
6445                if (left == null) {
6446                        columnObject = stmt.getVariableName();
6447                } else if (left.getExpressionType() == EExpressionType.simple_object_name_t) {
6448                        columnObject = left.getObjectOperand();
6449                }
6450                if (columnObject != null) {
6451                        TableColumn tableColumn = null;
6452                        if (columnObject.getDbObjectType() == EDbObjectType.variable || stmt.getVariableName() != null) {
6453                                Variable tableModel;
6454                                if (columnObject.toString().indexOf(".") != -1) {
6455                                        List<String> splits = SQLUtil.parseNames(columnObject.toString());
6456                                        tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6457                                } else {
6458                                        tableModel = modelFactory.createVariable(columnObject);
6459                                }
6460                                tableModel.setCreateTable(true);
6461                                tableModel.setSubType(SubType.record);
6462                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6463                                        tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6464                                } else {
6465                                        tableColumn = tableModel.getColumns().get(0);
6466                                }
6467                        } else {
6468                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6469                                if (splits.size() > 1) {
6470                                        Table tableModel = modelManager
6471                                                        .getTableByName(DlineageUtil.getTableFullName(splits.get(splits.size() - 2)));
6472                                        if (tableModel == null) {
6473                                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
6474                                                String variableString = splits.get(splits.size() - 2).toString();
6475                                                if (variableString.startsWith(":")) {
6476                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
6477                                                }
6478                                                if (!SQLUtil.isEmpty(procedureName)) {
6479                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
6480                                                }
6481                                                tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
6482                                        }
6483                                        if (tableModel != null) {
6484                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6485                                        }
6486                                }
6487                        }
6488
6489                        if (tableColumn != null && right != null) {
6490                                Transform transform = new Transform();
6491                                transform.setType(Transform.EXPRESSION);
6492                                transform.setCode(right);
6493                                tableColumn.setTransform(transform);;
6494                                columnsInExpr visitor = new columnsInExpr();
6495                                right.inOrderTraverse(visitor);
6496                                List<TObjectName> objectNames = visitor.getObjectNames();
6497                                List<TParseTreeNode> functions = visitor.getFunctions();
6498                                List<TParseTreeNode> constants = visitor.getConstants();
6499                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6500
6501                                if (functions != null && !functions.isEmpty()) {
6502                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6503                                }
6504                                if (subquerys != null && !subquerys.isEmpty()) {
6505                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6506                                }
6507                                if (objectNames != null && !objectNames.isEmpty()) {
6508                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6509                                }
6510                                if (constants != null && !constants.isEmpty()) {
6511                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6512                                }
6513                        }
6514                }
6515        }
6516
6517        private void analyzeOpenForStmt(TOpenforStmt stmt) {
6518                if (stmt.getSubquery() == null) {
6519                        return;
6520                }
6521
6522                Variable cursorTempTable = modelFactory.createCursor(stmt);
6523                cursorTempTable.setVariable(true);
6524                cursorTempTable.setSubType(SubType.cursor);
6525                modelManager.bindCursorModel(stmt, cursorTempTable);
6526                analyzeSelectStmt(stmt.getSubquery());
6527
6528                TableColumn cursorColumn = null;
6529                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6530                        TObjectName starColumn = new TObjectName();
6531                        starColumn.setString("*");
6532                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6533                } else {
6534                        cursorColumn = cursorTempTable.getColumns().get(0);
6535                }
6536
6537                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6538                if (resultSetModel != null) {
6539                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6540                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6541                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6542                                dataflowRelation.setEffectType(EffectType.cursor);
6543                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6544                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6545                        }
6546                }
6547        }
6548
6549        private void analyzeCursorDeclStmt(TCursorDeclStmt stmt) {
6550                if (stmt.getSubquery() == null) {
6551                        return;
6552                }
6553
6554                Variable cursorTempTable = modelFactory.createCursor(stmt);
6555                cursorTempTable.setVariable(true);
6556                cursorTempTable.setSubType(SubType.cursor);
6557                modelManager.bindCursorModel(stmt, cursorTempTable);
6558                analyzeSelectStmt(stmt.getSubquery());
6559                
6560                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6561                if(resultSetModel!=null && resultSetModel.isDetermined()) {
6562                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6563                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6564                                TObjectName columnName = new TObjectName();
6565                                columnName.setString(resultColumn.getName());
6566                                TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, columnName, true);           
6567                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6568                                dataflowRelation.setEffectType(EffectType.cursor);
6569                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6570                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6571                        }
6572                }
6573                else {
6574                        TableColumn cursorColumn = null;
6575                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6576                                TObjectName starColumn = new TObjectName();
6577                                starColumn.setString("*");
6578                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6579                        } else {
6580                                cursorColumn = cursorTempTable.getColumns().get(0);
6581                        }
6582                        if (resultSetModel != null) {
6583                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6584                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6585                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6586                                        dataflowRelation.setEffectType(EffectType.cursor);
6587                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6588                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6589                                }
6590                        }
6591                }
6592                
6593        }
6594
6595        private void analyzeDb2Declare(TDb2SqlVariableDeclaration stmt) {
6596                TDeclareVariableList variables = stmt.getVariables();
6597                if (variables == null) {
6598                        return;
6599                }
6600                for (int i = 0; i < variables.size(); i++) {
6601                        TDeclareVariable variable = variables.getDeclareVariable(i);
6602                        if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
6603
6604
6605                                TObjectName tableName = variable.getVariableName();
6606                                TTableElementList columns = variable.getTableTypeDefinitions();
6607
6608                                Table tableModel = modelFactory.createTableByName(tableName, true);
6609                                tableModel.setCreateTable(true);
6610                                String procedureParent = getProcedureParentName(stmt);
6611                                if (procedureParent != null) {
6612                                        tableModel.setParent(procedureParent);
6613                                }
6614                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
6615
6616                                for (int j = 0; j < columns.size(); j++) {
6617                                        TTableElement tableElement = columns.getTableElement(j);
6618                                        TColumnDefinition column = tableElement.getColumnDefinition();
6619                                        if (column != null && column.getColumnName() != null) {
6620                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
6621                                        }
6622                                }
6623                        } else if (variable.getVariableName() != null) {
6624                                Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
6625                                cursorVariable.setCreateTable(true);
6626                                cursorVariable.setSubType(SubType.record);
6627                                if(variable.getDatatype()!=null && isSimpleDataType(variable.getDatatype())){
6628                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
6629                                                        variable.getVariableName(), true);
6630                                }
6631                                else {
6632                                        TObjectName variableProperties = new TObjectName();
6633                                        variableProperties.setString("*");
6634                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
6635                                                        variableProperties, true);
6636                                }
6637                        }
6638                }
6639        }
6640
6641        private void analyzeMssqlDeclare(TMssqlDeclare stmt) {
6642                if (stmt.getDeclareType() == EDeclareType.variable) {
6643                        TDeclareVariableList variables = stmt.getVariables();
6644                        if (variables == null) {
6645                                return;
6646                        }
6647                        for (int i = 0; i < variables.size(); i++) {
6648                                TDeclareVariable variable = variables.getDeclareVariable(i);
6649                                if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
6650
6651
6652                                        TObjectName tableName = variable.getVariableName();
6653                                        TTableElementList columns = variable.getTableTypeDefinitions();
6654
6655                                        Table tableModel = modelFactory.createTableByName(tableName, true);
6656                                        tableModel.setCreateTable(true);
6657                                        String procedureParent = getProcedureParentName(stmt);
6658                                        if (procedureParent != null) {
6659                                                tableModel.setParent(procedureParent);
6660                                        }
6661                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
6662
6663                                        for (int j = 0; j < columns.size(); j++) {
6664                                                TTableElement tableElement = columns.getTableElement(j);
6665                                                TColumnDefinition column = tableElement.getColumnDefinition();
6666                                                if (column != null && column.getColumnName() != null) {
6667                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
6668                                                }
6669                                        }
6670                                } else if (variable.getVariableName() != null) {
6671                                        Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
6672                                        cursorVariable.setCreateTable(true);
6673                                        cursorVariable.setSubType(SubType.record);
6674                                        TableColumn variableProperty = null;
6675                                        if (variable.getDatatype() != null && isSimpleDataType(variable.getDatatype())) {
6676                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variable.getVariableName(),
6677                                                                true);
6678                                        } else {
6679                                                TObjectName variableProperties = new TObjectName();
6680                                                variableProperties.setString("*");
6681                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variableProperties, true);
6682                                        }
6683                                        
6684                                        if (variable.getDefaultValue() != null && variable.getDefaultValue().getSubQuery() != null) {
6685                                                analyzeSelectStmt(variable.getDefaultValue().getSubQuery());
6686                                                ResultSet resultSetModel = (ResultSet) modelManager
6687                                                                .getModel(variable.getDefaultValue().getSubQuery());
6688                                                if (variableProperty != null && resultSetModel != null) {
6689                                                        for (ResultColumn column : resultSetModel.getColumns()) {
6690                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6691                                                                dataflowRelation.setEffectType(EffectType.select);
6692                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
6693                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6694                                                        }
6695                                                }
6696                                        }
6697                                }
6698                        }
6699                } else if (stmt.getDeclareType() == EDeclareType.cursor) {
6700                        Variable cursorTempTable = modelFactory.createCursor(stmt);
6701                        cursorTempTable.setVariable(true);
6702                        cursorTempTable.setSubType(SubType.cursor);
6703                        modelManager.bindCursorModel(stmt, cursorTempTable);
6704                        analyzeSelectStmt(stmt.getSubquery());
6705                        ResultSet resultSetModel = (ResultSet)modelManager.getModel(stmt.getSubquery());
6706                        if (resultSetModel != null && resultSetModel.isDetermined()) {
6707                                for(ResultColumn resultColumn: resultSetModel.getColumns()){
6708                                        TObjectName starColumn = new TObjectName();
6709                                        starColumn.setString(resultColumn.getName());
6710                                        TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6711                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6712                                        dataflowRelation.setEffectType(EffectType.cursor);
6713                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6714                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6715                                }
6716                        }
6717                        else {
6718                                TableColumn cursorColumn = null;
6719                                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6720                                        TObjectName starColumn = new TObjectName();
6721                                        starColumn.setString("*");
6722                                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6723                                        cursorColumn.setShowStar(false);
6724                                        cursorColumn.setExpandStar(true);
6725                                } else {
6726                                        cursorColumn = cursorTempTable.getColumns().get(0);
6727                                }
6728
6729                                if (resultSetModel != null) {
6730                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6731                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6732                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6733                                                dataflowRelation.setEffectType(EffectType.cursor);
6734                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6735                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6736                                        }
6737                                }
6738                        }
6739                }
6740        }
6741
6742
6743        private boolean analyzeMssqlJsonDeclare(TMssqlDeclare stmt, String jsonName, Table jsonTable) {
6744                TDeclareVariableList variables = stmt.getVariables();
6745                if (variables == null) {
6746                        return false;
6747                }
6748                for (int i = 0; i < variables.size(); i++) {
6749                        TDeclareVariable variable = variables.getDeclareVariable(i);
6750                        TObjectName variableName = variable.getVariableName();
6751                        if (DlineageUtil.getIdentifierNormalTableName(variableName.toString())
6752                                        .equals(DlineageUtil.getIdentifierNormalTableName(jsonName))) {
6753                                if (variable.getDefaultValue() != null) {
6754                                        Table variableTable = modelFactory.createJsonVariable(variableName);
6755                                        variableTable.setVariable(true);
6756                                        variableTable.setSubType(SubType.scalar);
6757                                        variableTable.setCreateTable(true);
6758                                        TableColumn property = modelFactory.createVariableProperty(variableTable, variable);
6759
6760                                        for (int j = 0; j < jsonTable.getColumns().size(); j++) {
6761                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6762                                                relation.setEffectType(EffectType.select);
6763                                                relation.setTarget(new TableColumnRelationshipElement(jsonTable.getColumns().get(j)));
6764                                                relation.addSource(new TableColumnRelationshipElement(property));
6765                                        }
6766                                }
6767                                return true;
6768                        }
6769                }
6770                return false;
6771        }
6772
6773        private void analyzeCreateTableStmt(TCreateTableSqlStatement stmt) {
6774                if (stmt.getCloneSourceTable() != null) {
6775                        analyzeCloneTableStmt(stmt);
6776                        return;
6777                }
6778
6779                TTable table = stmt.getTargetTable();
6780
6781                boolean hasDefinition = false;
6782
6783                if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
6784                        hasDefinition = true;
6785                }
6786
6787                if (table != null) {
6788                        Table tableModel = modelFactory.createTableFromCreateDDL(table, hasDefinition || (stmt.getSubQuery() == null && hasDefinition)
6789                                        || (stmt.getSubQuery()!=null && stmt.getSubQuery().getSetOperatorType() == ESetOperatorType.none && stmt.getSubQuery().getResultColumnList().toString().indexOf("*") == -1));
6790                        if (stmt.isExternal()) {
6791                                tableModel.setExternal(true);
6792                        }
6793
6794                        if (stmt.getSubQuery() != null) {
6795                                Process process = modelFactory.createProcess(stmt);
6796                                tableModel.addProcess(process);
6797                        }
6798
6799                        String procedureParent = getProcedureParentName(stmt);
6800                        if (procedureParent != null) {
6801                                tableModel.setParent(procedureParent);
6802                        }
6803
6804                        if (hasDefinition) {
6805                                if(stmt.getSubQuery()!=null) {
6806                                        TSelectSqlStatement subquery = stmt.getSubQuery();
6807                                        analyzeSelectStmt(subquery);
6808                                        Process process = modelFactory.createProcess(stmt);
6809                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
6810                                        if(resultSetModel.isDetermined()){
6811                                                tableModel.setFromDDL(true);
6812                                        }
6813                                        if (resultSetModel != null) {
6814                                                int resultSetSize = resultSetModel.getColumns().size();
6815                                                int stmtColumnSize = stmt.getColumnList().size();
6816                                                int j = 0;
6817                                                int tableColumnSize = stmtColumnSize;
6818                                                if (resultSetModel.isDetermined() && resultSetSize > tableColumnSize) {
6819                                                        tableColumnSize = resultSetSize;
6820                                                }
6821                                                for (int i = 0; i < tableColumnSize && j < resultSetSize; i++) {
6822                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
6823                                                        if (i < stmtColumnSize) {
6824                                                                TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
6825                                                                
6826                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
6827                                                                        j++;
6828                                                                } else {
6829                                                                        if (resultSetSize - j == stmt.getColumnList().size() - i) {
6830                                                                                j++;
6831                                                                        }
6832                                                                }
6833
6834                                                                if (alias != null) {
6835                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
6836                                                                        if (resultColumn != null) {
6837                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6838                                                                                relation.setEffectType(EffectType.create_table);
6839                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6840                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6841                                                                                relation.setProcess(process);
6842                                                                        }
6843                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
6844                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
6845                                                                                        (TObjectName) resultColumn.getColumnObject(), true);
6846                                                                        if (resultColumn != null) {
6847                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6848                                                                                relation.setEffectType(EffectType.create_table);
6849                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6850                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6851                                                                                relation.setProcess(process);
6852                                                                        }
6853                                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
6854                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
6855                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), true);
6856                                                                        ResultColumn column = (ResultColumn) modelManager
6857                                                                                        .getModel(resultColumn.getColumnObject());
6858                                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
6859                                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
6860                                                                        }
6861                                                                        if (resultColumn != null) {
6862                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6863                                                                                relation.setEffectType(EffectType.create_table);
6864                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6865                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6866                                                                                relation.setProcess(process);
6867                                                                        }
6868                                                                }
6869                                                        }
6870                                                        else if(resultSetModel.isDetermined()){
6871                                                                TObjectName tableName = new TObjectName();
6872                                                                tableName.setString(resultColumn.getName());
6873                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, tableName, true);
6874                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6875                                                                relation.setEffectType(EffectType.create_table);
6876                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6877                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6878                                                                relation.setProcess(process);
6879                                                                j++;
6880                                                        }
6881                                                }
6882                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
6883                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
6884                                                        impactRelation.setEffectType(EffectType.create_table);
6885                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
6886                                                                        resultSetModel.getRelationRows()));
6887                                                        impactRelation.setTarget(
6888                                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
6889                                                }
6890                                        }
6891
6892                                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
6893                                                        && subquery.getValueClause().getValueRows().size() == stmt.getColumnList().size()) {
6894                                                for (int i = 0; i < stmt.getColumnList().size(); i++) {
6895                                                        TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
6896
6897                                                        if (alias != null) {
6898                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
6899
6900                                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
6901
6902                                                                columnsInExpr visitor = new columnsInExpr();
6903                                                                expression.inOrderTraverse(visitor);
6904                                                                List<TObjectName> objectNames = visitor.getObjectNames();
6905                                                                List<TParseTreeNode> functions = visitor.getFunctions();
6906
6907                                                                if (functions != null && !functions.isEmpty()) {
6908                                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
6909
6910                                                                }
6911
6912                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6913                                                                if (subquerys != null && !subquerys.isEmpty()) {
6914                                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
6915                                                                }
6916
6917                                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
6918                                                                List<TParseTreeNode> constants = visitor.getConstants();
6919                                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
6920                                                        }
6921                                                }
6922                                        }
6923                                        
6924                                        return;
6925                                }
6926                                else {
6927                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
6928                                                TColumnDefinition column = stmt.getColumnList().getColumn(i);
6929                                                if (column.getDatatype() != null && column.getDatatype().getTypeOfList() != null
6930                                                                && column.getDatatype().getTypeOfList().getColumnDefList() != null) {
6931                                                        for (int j = 0; j < column.getDatatype().getTypeOfList().getColumnDefList().size(); j++) {
6932                                                                TObjectName columnName = new TObjectName();
6933                                                                if (column.getDatatype().getDataType() == EDataType.array_t) {
6934//                                                                      columnName.setString(column.getColumnName().getColumnNameOnly() + ".array."
6935//                                                                                      + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
6936//                                                                                                      .getColumnName().getColumnNameOnly());
6937                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
6938                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
6939                                                                                        .getColumnName().getColumnNameOnly());
6940                                                                } else {
6941                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
6942                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
6943                                                                                                        .getColumnName().getColumnNameOnly());
6944                                                                }
6945                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName,
6946                                                                                hasDefinition);
6947                                                                tableColumn.setStruct(true);
6948                                                                tableColumn.setColumnIndex(i);
6949                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
6950                                                                
6951                                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
6952                                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
6953                                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
6954                                                                        crudRelationship.setEffectType(EffectType.create_table);
6955                                                                }
6956                                                        }
6957                                                        continue;
6958                                                }
6959                                                if (column.getDatatype() != null && column.getDatatype().getColumnDefList() != null) {
6960                                                        Stack<TColumnDefinition> columnPaths = new Stack<TColumnDefinition>();
6961                                                        flattenStructColumns(hasDefinition, tableModel, column, columnPaths, i);
6962                                                        continue;
6963                                                }
6964                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column.getColumnName(),
6965                                                                hasDefinition);
6966                                                if (tableColumn == null) {
6967                                                        continue;
6968                                                }
6969                                                
6970                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
6971                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
6972                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
6973                                                        crudRelationship.setEffectType(EffectType.create_table);
6974                                                }
6975                                                
6976                                                if (column.getDatatype() != null && column.getDatatype().getDataType() == EDataType.variant_t) {
6977                                                        if (tableColumn != null) {
6978                                                                tableColumn.setVariant(true);
6979                                                        }
6980                                                }
6981                                                if (column.getDatatype() != null) {
6982                                                        String dType = column.getDatatype().getDataTypeName();
6983                                                        if ((!SQLUtil.isEmpty(dType)) && (dType.indexOf("_") > 0)) {
6984                                                                dType = dType.split("_")[0];
6985                                                        }
6986                                                        tableColumn.setDataType(dType);
6987                                                }
6988                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
6989                                        }
6990                                }
6991                        }
6992
6993                        if (stmt.getExternalTableOption("DATA_SOURCE") != null) {
6994                                String dataSourceName = stmt.getExternalTableOption("DATA_SOURCE");
6995                                Table dataSource = modelManager.getTableByName(DlineageUtil.getTableFullName(dataSourceName));
6996                                if (dataSource != null) {
6997                                        TableColumn dataSourceColumn = dataSource.getColumns().get(0);
6998                                        for (int i = 0; i < tableModel.getColumns().size(); i++) {
6999                                                TableColumn column = tableModel.getColumns().get(i);
7000                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7001                                                relation.setTarget(new TableColumnRelationshipElement(column));
7002                                                relation.addSource(new TableColumnRelationshipElement(dataSourceColumn));
7003
7004                                                appendTableColumnToSQLEnv(tableModel, column);
7005                                        }
7006                                }
7007                        }
7008
7009                        if (stmt.getSubQuery() != null) {
7010
7011                                analyzeSelectStmt(stmt.getSubQuery());
7012
7013                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
7014                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
7015                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
7016                                        impactRelation.setEffectType(EffectType.create_table);
7017                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
7018                                                        resultSetModel.getRelationRows()));
7019                                        impactRelation.setTarget(
7020                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7021                                }
7022                        }
7023
7024                        if (stmt.getSubQuery() != null && !stmt.getSubQuery().isCombinedQuery()) {
7025                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
7026                                                .getModel(stmt.getSubQuery().getResultColumnList());
7027                                if(resultSetModel.isDetermined()){
7028                                        tableModel.setDetermined(true);
7029                                        tableModel.setFromDDL(true);
7030                                }
7031                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7032                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7033                                        if (resultSetModel.isDetermined() && resultColumn.getName().endsWith("*")) {
7034                                                continue;
7035                                        }
7036
7037                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
7038                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
7039
7040                                                TAliasClause alias = columnObject.getAliasClause();
7041                                                if (alias != null && alias.getAliasName() != null) {
7042                                                        TableColumn tableColumn = null;
7043                                                        if (!hasDefinition) {
7044                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
7045                                                                                !hasDefinition);
7046                                                        } else {
7047                                                                tableColumn = tableModel.getColumns().get(i);
7048                                                        }
7049
7050                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7051                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7052                                                        }
7053
7054                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7055                                                        relation.setEffectType(EffectType.create_table);
7056                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7057                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7058                                                        Process process = modelFactory.createProcess(stmt);
7059                                                        relation.setProcess(process);
7060                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
7061                                                        TableColumn tableColumn = null;
7062                                                        TObjectName columnObj = columnObject.getFieldAttr();
7063                                                        if(columnObj == null) {
7064                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
7065                                                        }
7066                                                        if ((columnObj == null || columnObj.toString().endsWith("*")) && !resultColumn.getName().endsWith("*")) {
7067                                                                columnObj = new TObjectName();
7068                                                                columnObj.setString(resultColumn.getName());
7069                                                        }
7070
7071                                                        if(columnObj == null){
7072                                                                logger.info("Can't handle column " + resultColumn.getName());
7073                                                                continue;
7074                                                        }
7075
7076                                                        if (!hasDefinition) {
7077                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
7078                                                                                !hasDefinition);
7079                                                                if (tableColumn == null) {
7080                                                                        if (tableModel.getColumns().isEmpty()) {
7081                                                                                logger.info("Add table " + tableModel.getName() + " column " + columnObj.toString() + " failed");
7082                                                                        }
7083                                                                        else if (resultColumn.getName().endsWith("*")) {
7084                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
7085                                                                                        tableColumn = tableModel.getColumns().get(j);
7086                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7087                                                                                        relation.setEffectType(EffectType.create_table);
7088                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7089                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7090                                                                                        if (tableColumn.getName().endsWith("*")
7091                                                                                                        && resultColumn.getName().endsWith("*")) {
7092                                                                                                tableModel.setStarStmt("create_table");
7093                                                                                        }
7094                                                                                        Process process = modelFactory.createProcess(stmt);
7095                                                                                        relation.setProcess(process);
7096                                                                                }
7097                                                                        }
7098                                                                        continue;
7099                                                                }
7100                                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7101                                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7102                                                                }
7103
7104                                                                Object model = modelManager
7105                                                                                .getModel(resultColumn.getColumnObject());
7106                                                                if (model instanceof ResultColumn) {
7107                                                                        ResultColumn column = (ResultColumn) model;
7108                                                                        if (tableColumn.getName().endsWith("*") && column != null
7109                                                                                        && !column.getStarLinkColumns().isEmpty()) {
7110                                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7111                                                                        }
7112
7113                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7114                                                                        relation.setEffectType(EffectType.create_table);
7115                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7116                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7117                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
7118                                                                                tableModel.setStarStmt("create_table");
7119                                                                        }
7120                                                                        Process process = modelFactory.createProcess(stmt);
7121                                                                        relation.setProcess(process);
7122                                                                }
7123                                                                else if(model instanceof LinkedHashMap) {
7124                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
7125                                                                        LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
7126                                                                        if (columnObj.toString().endsWith("*")) {
7127                                                                                for (String key : resultColumns.keySet()) {
7128                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
7129                                                                                        DataFlowRelationship relation = modelFactory
7130                                                                                                        .createDataFlowRelation();
7131                                                                                        relation.setEffectType(EffectType.create_table);
7132                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7133                                                                                        relation.addSource(
7134                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
7135                                                                                        Process process = modelFactory.createProcess(stmt);
7136                                                                                        relation.setProcess(process);
7137                                                                                }
7138                                                                        } else if (resultColumns.containsKey(columnName)) {
7139                                                                                ResultColumn column = resultColumns.get(columnName);
7140                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
7141                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7142                                                                                relation.setEffectType(EffectType.create_table);
7143                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7144                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
7145                                                                                Process process = modelFactory.createProcess(stmt);
7146                                                                                relation.setProcess(process);
7147                                                                        }
7148                                                                }
7149                                                        } else {
7150                                                                if (resultColumn.getName().endsWith("*")) {
7151                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
7152                                                                                tableColumn = tableModel.getColumns().get(j);
7153                                                                                ResultColumn column = (ResultColumn) modelManager
7154                                                                                                .getModel(resultColumn.getColumnObject());
7155                                                                                if (tableColumn.getName().endsWith("*") && column != null
7156                                                                                                && !column.getStarLinkColumns().isEmpty()) {
7157                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7158                                                                                }
7159
7160                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7161                                                                                relation.setEffectType(EffectType.create_table);
7162                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7163                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7164                                                                                if (tableColumn.getName().endsWith("*")
7165                                                                                                && resultColumn.getName().endsWith("*")) {
7166                                                                                        tableModel.setStarStmt("create_table");
7167                                                                                        ;
7168                                                                                }
7169                                                                                Process process = modelFactory.createProcess(stmt);
7170                                                                                relation.setProcess(process);
7171                                                                        }
7172                                                                } else {
7173                                                                        tableColumn = tableModel.getColumns().get(i);
7174                                                                        Object model = modelManager
7175                                                                                        .getModel(resultColumn.getColumnObject());
7176                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
7177                                                                        if(model instanceof LinkedHashMap) {
7178                                                                                LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
7179                                                                                if (resultColumns.size() == tableModel.getColumns().size()) {
7180                                                                                        int j = 0;
7181                                                                                        for (String key : resultColumns.keySet()) {
7182                                                                                                if (j == i) {
7183                                                                                                        ResultColumn column = resultColumns.get(key);
7184                                                                                                        DataFlowRelationship relation = modelFactory
7185                                                                                                                        .createDataFlowRelation();
7186                                                                                                        relation.setEffectType(EffectType.create_table);
7187                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7188                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
7189                                                                                                        Process process = modelFactory.createProcess(stmt);
7190                                                                                                        relation.setProcess(process);
7191                                                                                                }
7192                                                                                                j++;
7193                                                                                        }
7194                                                                                } else if (resultColumns.containsKey(columnName)) {
7195                                                                                        ResultColumn column = resultColumns.get(columnName);
7196                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
7197                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7198                                                                                        relation.setEffectType(EffectType.create_table);
7199                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7200                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
7201                                                                                        Process process = modelFactory.createProcess(stmt);
7202                                                                                        relation.setProcess(process);
7203                                                                                } else {
7204                                                                                        throw new UnsupportedOperationException("Can't handle this star case.");
7205                                                                                }
7206                                                                        }
7207                                                                        else if (model instanceof ResultColumn) {
7208                                                                                ResultColumn column = (ResultColumn) modelManager
7209                                                                                                .getModel(resultColumn.getColumnObject());
7210                                                                                if (tableColumn.getName().endsWith("*") && column != null
7211                                                                                                && !column.getStarLinkColumns().isEmpty()) {
7212                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7213                                                                                }
7214
7215                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7216                                                                                relation.setEffectType(EffectType.create_table);
7217                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7218                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7219                                                                                if (tableColumn.getName().endsWith("*")
7220                                                                                                && resultColumn.getName().endsWith("*")) {
7221                                                                                        tableModel.setStarStmt("create_table");
7222                                                                                }
7223                                                                                Process process = modelFactory.createProcess(stmt);
7224                                                                                relation.setProcess(process);
7225                                                                        }
7226                                                                }
7227                                                        }
7228                                                } else {
7229                                                        TableColumn tableColumn = null;
7230                                                        if (!hasDefinition) {
7231                                                                TObjectName columnName = new TObjectName();
7232                                                                columnName.setString(resultColumn.getColumnObject().toString());
7233                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName, !hasDefinition);
7234                                                                if(tableColumn == null){
7235                                                                        continue;
7236                                                                }
7237                                                        } else {
7238                                                                tableColumn = tableModel.getColumns().get(i);
7239                                                        }
7240                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
7241                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
7242                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7243                                                        }
7244
7245                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7246                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7247                                                        }
7248
7249                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7250                                                        relation.setEffectType(EffectType.create_table);
7251                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7252                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7253                                                        Process process = modelFactory.createProcess(stmt);
7254                                                        relation.setProcess(process);
7255                                                }
7256                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
7257                                                TableColumn tableColumn = null;
7258                                                if (!hasDefinition) {
7259                                                        tableColumn = modelFactory.createTableColumn(tableModel,
7260                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
7261                                                } else {
7262                                                        tableColumn = tableModel.getColumns().get(i);
7263                                                }
7264
7265                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7266                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7267                                                }
7268
7269                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7270                                                relation.setEffectType(EffectType.create_table);
7271                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7272                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7273                                                Process process = modelFactory.createProcess(stmt);
7274                                                relation.setProcess(process);
7275                                        }
7276                                }
7277                        } else if (stmt.getSubQuery() != null) {
7278                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(stmt.getSubQuery());
7279                                if(resultSetModel.isDetermined()){
7280                                        tableModel.setDetermined(true);
7281                                        tableModel.setFromDDL(true);
7282                                }
7283                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7284                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7285                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
7286                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
7287
7288                                                TAliasClause alias = columnObject.getAliasClause();
7289                                                if (alias != null && alias.getAliasName() != null) {
7290                                                        TableColumn tableColumn = null;
7291                                                        if (!hasDefinition) {
7292                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
7293                                                                                !hasDefinition);
7294                                                                if (tableColumn == null) {
7295                                                                        continue;
7296                                                                }
7297                                                        } else {
7298                                                                tableColumn = tableModel.getColumns().get(i);
7299                                                        }
7300
7301                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7302                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7303                                                        }
7304
7305                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7306                                                        relation.setEffectType(EffectType.create_table);
7307                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7308                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7309                                                        Process process = modelFactory.createProcess(stmt);
7310                                                        relation.setProcess(process);
7311                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
7312                                                        TableColumn tableColumn = null;
7313                                                        TObjectName columnObj = columnObject.getFieldAttr();
7314                                                        if(columnObj == null) {
7315                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
7316                                                        }
7317                                                        if (!hasDefinition) {
7318                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
7319                                                                                !hasDefinition);
7320                                                                if (tableColumn == null) {
7321                                                                        continue;
7322                                                                }
7323                                                        } else {
7324                                                                tableColumn = tableModel.getColumns().get(i);
7325                                                        }
7326                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
7327                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
7328                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7329                                                        }
7330
7331                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7332                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7333                                                        }
7334
7335                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7336                                                        relation.setEffectType(EffectType.create_table);
7337                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7338                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7339                                                        Process process = modelFactory.createProcess(stmt);
7340                                                        relation.setProcess(process);
7341                                                } else {
7342                                                        ErrorInfo errorInfo = new ErrorInfo();
7343                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7344                                                        errorInfo.setErrorMessage("Can't handle the table column " + columnObject.toString());
7345                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(
7346                                                                        columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
7347                                                                        ModelBindingManager.getGlobalHash()));
7348                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
7349                                                                        columnObject.getEndToken().columnNo + columnObject.getEndToken().getAstext().length(),
7350                                                                        ModelBindingManager.getGlobalHash()));
7351                                                        errorInfos.add(errorInfo);
7352                                                        continue;
7353                                                }
7354                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
7355                                                TableColumn tableColumn = null;
7356                                                if (!hasDefinition) {
7357                                                        tableColumn = modelFactory.createTableColumn(tableModel,
7358                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
7359                                                } else {
7360                                                        tableColumn = tableModel.getColumns().get(i);
7361                                                }
7362
7363                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7364                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7365                                                }
7366
7367                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7368                                                relation.setEffectType(EffectType.create_table);
7369                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7370                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7371                                                Process process = modelFactory.createProcess(stmt);
7372                                                relation.setProcess(process);
7373                                        }
7374                                }
7375                        } else if (stmt.getLikeTableName() != null) {
7376                                Table likeTableModel = modelFactory.createTableByName(stmt.getLikeTableName());
7377
7378                                if(likeTableModel.isCreateTable()){
7379                                        for(TableColumn column: likeTableModel.getColumns()){
7380                                                TObjectName tableColumn = new TObjectName();
7381                                                tableColumn.setString(column.getName());
7382                                                TableColumn createTableColumn = modelFactory.createTableColumn(tableModel, tableColumn, true);
7383                                                createTableColumn.setPrimaryKey(column.getPrimaryKey());
7384                                                createTableColumn.setForeignKey(column.getForeignKey());
7385                                                createTableColumn.setIndexKey(column.getIndexKey());
7386                                                createTableColumn.setUnqiueKey(column.getUnqiueKey());
7387                                                createTableColumn.setDataType(column.getDataType());
7388                                        }
7389                                        tableModel.setCreateTable(true);
7390                                }
7391
7392//                              Process process = modelFactory.createProcess(stmt);
7393//                              process.setType("Like Table");
7394//                              tableModel.addProcess(process);
7395//
7396//
7397//                              DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7398//                              relation.setEffectType(EffectType.like_table);
7399//                              relation.setTarget(
7400//                                              new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7401//                              relation.addSource(
7402//                                              new RelationRowsRelationshipElement<TableRelationRows>(likeTableModel.getRelationRows()));
7403//                              relation.setProcess(process);
7404                        }
7405
7406                        if (stmt.getStageLocation() != null && stmt.getStageLocation().getStageName() != null) {
7407                                tableModel
7408                                                .setLocation(DlineageUtil.getTableFullName(stmt.getStageLocation().getStageName().toString()));
7409                                Process process = modelFactory.createProcess(stmt);
7410                                process.setType("Create External Table");
7411                                tableModel.addProcess(process);
7412                                Table stage = modelManager.getTableByName(DlineageUtil.getTableFullName(tableModel.getLocation()));
7413                                if (stage == null) {
7414                                        stage = modelManager.getTableByName(
7415                                                        DlineageUtil.getTableFullName(stmt.getStageLocation().toString().replaceFirst("@", "")));
7416                                }
7417                                if (stage == null) {
7418                                        stage = modelFactory.createTableByName(stmt.getStageLocation().toString().replaceFirst("@", ""),
7419                                                        false);
7420                                        stage.setCreateTable(false);
7421                                        stage.setStage(true);
7422                                        if (stmt.getStageLocation().getPath() != null) {
7423                                                stage.setLocation(stmt.getStageLocation().getPath().toString());
7424                                                TObjectName location = new TObjectName();
7425                                                location.setString(stmt.getStageLocation().getPath().toString());
7426                                                modelFactory.createStageLocation(stage, location);
7427                                        } else if (stmt.getRegex_pattern() != null) {
7428                                                stage.setLocation(stmt.getRegex_pattern());
7429                                                TObjectName location = new TObjectName();
7430                                                location.setString(stmt.getRegex_pattern());
7431                                                modelFactory.createStageLocation(stage, location);
7432                                        } else {
7433                                                stage.setLocation("unknownPath");
7434                                                TObjectName location = new TObjectName();
7435                                                location.setString("unknownPath");
7436                                                modelFactory.createStageLocation(stage, location);
7437                                        }
7438                                }
7439                                if (stage != null && !stage.getColumns().isEmpty()) {
7440                                        if (!tableModel.getColumns().isEmpty()) {
7441                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
7442                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7443                                                        relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
7444                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
7445                                                        relation.setProcess(process);
7446                                                }
7447                                        } else {
7448                                                TObjectName starColumn = new TObjectName();
7449                                                starColumn.setString("*");
7450                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
7451                                                tableColumn.setExpandStar(false);
7452                                                tableColumn.setShowStar(true);
7453                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7454                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
7455                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7456                                                relation.setProcess(process);
7457                                        }
7458                                }
7459                        } else if (stmt.getTableOptions() != null && !stmt.getTableOptions().isEmpty()) {
7460                                for (int i = 0; i < stmt.getTableOptions().size(); i++) {
7461                                        TCreateTableOption createTableOption = stmt.getTableOptions().get(i);
7462                                        if (createTableOption.getCreateTableOptionType() != ECreateTableOption.etoBigQueryExternal)
7463                                                continue;
7464                                        List<String> uris = createTableOption.getUris();
7465                                        if (uris == null || uris.isEmpty())
7466                                                continue;
7467                                        Process process = modelFactory.createProcess(stmt);
7468                                        process.setType("Create External Table");
7469                                        tableModel.addProcess(process);
7470                                        if (tableModel.getColumns().isEmpty()) {
7471                                                TObjectName starColumn = new TObjectName();
7472                                                starColumn.setString("*");
7473                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
7474                                                tableColumn.setExpandStar(false);
7475                                                tableColumn.setShowStar(true);
7476                                        }
7477                                        for (String uri : uris) {
7478                                                Table uriFile = modelFactory.createTableByName(uri, true);
7479                                                uriFile.setPath(true);
7480                                                uriFile.setFileFormat(createTableOption.getFormat());
7481                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
7482                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7483                                                        if (stmt.getColumnList() != null) {
7484                                                                TObjectName fileUri = new TObjectName();
7485                                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
7486                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7487                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7488                                                        } else {
7489                                                                TObjectName fileUri = new TObjectName();
7490                                                                fileUri.setString("*");
7491                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7492                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7493                                                        }
7494                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
7495                                                        relation.setProcess(process);
7496                                                }
7497                                                if (tableModel.getColumns().isEmpty()) {
7498                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7499                                                        TObjectName fileUri = new TObjectName();
7500                                                        fileUri.setString("*");
7501                                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7502                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7503                                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
7504                                                                        tableModel.getRelationRows()));
7505                                                        relation.setProcess(process);
7506                                                }
7507                                        }
7508                                }
7509                        } else if (stmt.getSubQuery() == null && stmt.getTableLocation() != null) {
7510                                Process process = modelFactory.createProcess(stmt);
7511                                process.setType("Create External Table");
7512                                tableModel.addProcess(process);
7513                                Table uriFile = modelFactory.createTableByName(stmt.getTableLocation(), true);
7514                                uriFile.setPath(true);
7515                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
7516                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7517                                        if (stmt.getColumnList() != null) {
7518                                                TObjectName fileUri = new TObjectName();
7519                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
7520                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7521                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7522                                        } else {
7523                                                TObjectName fileUri = new TObjectName();
7524                                                fileUri.setString("*");
7525                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7526                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7527                                        }
7528                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
7529                                        relation.setProcess(process);
7530                                }
7531                        }
7532
7533                        if (stmt.getTableConstraints() != null && stmt.getTableConstraints().size() > 0) {
7534                                for (int i = 0; i < stmt.getTableConstraints().size(); i++) {
7535                                        TConstraint createTableConstraint = stmt.getTableConstraints().getConstraint(i);
7536                                        TPTNodeList<TColumnWithSortOrder> keyNames = createTableConstraint.getColumnList();
7537                                        if (keyNames != null) {
7538                                                for (int k = 0; k < keyNames.size(); k++) {
7539                                                        TObjectName keyName = keyNames.getElement(k).getColumnName();
7540                                                        // Skip functional indexes (expression-based indexes) where columnName is null
7541                                                        if (keyName == null) {
7542                                                                continue;
7543                                                        }
7544                                                        TObjectName referencedTableName = createTableConstraint.getReferencedObject();
7545                                                        TObjectNameList referencedTableColumns = createTableConstraint.getReferencedColumnList();
7546
7547                                                        TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
7548                                                        if (createTableConstraint.getConstraint_type() == EConstraintType.primary_key) {
7549                                                                tableConstraint.setPrimaryKey(true);
7550                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.table_index) {
7551                                                                tableConstraint.setIndexKey(true);
7552                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.unique) {
7553                                                                tableConstraint.setUnqiueKey(true);
7554                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
7555                                                                tableConstraint.setForeignKey(true);
7556                                                                Table referencedTable = modelManager
7557                                                                                .getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
7558                                                                if (referencedTable == null) {
7559                                                                        referencedTable = modelFactory.createTableByName(referencedTableName);
7560                                                                }
7561
7562                                                                if (referencedTableColumns != null) {
7563                                                                        for (int j = 0; j < referencedTableColumns.size(); j++) {
7564                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
7565                                                                                                referencedTableColumns.getObjectName(j), false);
7566                                                                                if (tableColumn != null) {
7567                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7568                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
7569                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
7570                                                                                        relation.setEffectType(EffectType.foreign_key);
7571                                                                                        Process process = modelFactory.createProcess(stmt);
7572                                                                                        relation.setProcess(process);
7573                                                                                        if (this.option.isShowERDiagram()) {
7574                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
7575                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
7576                                                                                                erRelation
7577                                                                                                                .setTarget(new TableColumnRelationshipElement(tableConstraint));
7578                                                                                        }
7579                                                                                }
7580                                                                        }
7581                                                                }
7582                                                        }
7583                                                }
7584                                        }
7585                                }
7586                        }
7587
7588                        if (stmt.getHiveTablePartition() != null && stmt.getHiveTablePartition().getColumnDefList() != null) {
7589                                for (int i = 0; i < stmt.getHiveTablePartition().getColumnDefList().size(); i++) {
7590                                        TColumnDefinition column = stmt.getHiveTablePartition().getColumnDefList().getColumn(i);
7591                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
7592                                        appendTableColumnToSQLEnv(tableModel, column.getColumnName());
7593                                }
7594                        }
7595
7596                } else {
7597                        ErrorInfo errorInfo = new ErrorInfo();
7598                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7599                        errorInfo.setErrorMessage("Can't get target table. CreateTableSqlStatement is " + stmt.toString());
7600                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
7601                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
7602                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
7603                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
7604                                        ModelBindingManager.getGlobalHash()));
7605                        errorInfos.add(errorInfo);
7606                }
7607        }
7608
7609        protected void flattenStructColumns(boolean hasDefinition, Table tableModel, TColumnDefinition column,
7610                        Stack<TColumnDefinition> columnPaths, int index) {
7611                columnPaths.push(column);
7612                for (int j = 0; j < column.getDatatype().getColumnDefList().size(); j++) {
7613                        TColumnDefinition columnDefinition = column.getDatatype().getColumnDefList().getColumn(j);
7614                        if (columnDefinition.getDatatype().getColumnDefList() != null) {
7615                                flattenStructColumns(hasDefinition, tableModel, columnDefinition, columnPaths, index);
7616                        } else {
7617                                TObjectName columnName = new TObjectName();
7618                                columnName.setString(getColumnName(columnPaths, columnDefinition));
7619                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName, hasDefinition);
7620                                tableColumn.setColumnIndex(index);
7621                                tableColumn.setStruct(true);
7622                                
7623                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
7624                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
7625                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
7626                                        crudRelationship.setEffectType(EffectType.create_table);
7627                                }
7628                                
7629                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7630                        }
7631                }
7632                columnPaths.pop();
7633        }
7634
7635        private String getColumnName(Stack<TColumnDefinition> columnPaths, TColumnDefinition column) {
7636                StringBuilder buffer = new StringBuilder();
7637                Iterator<TColumnDefinition> iter = columnPaths.iterator();
7638                while(iter.hasNext()) {
7639                        buffer.append(iter.next().getColumnName().getColumnNameOnly()).append(".");
7640                }
7641                buffer.append(column.getColumnName().getColumnNameOnly());
7642                return buffer.toString();
7643        }
7644
7645        private void appendTableColumnToSQLEnv(Table tableModel, TableColumn tableColumn) {
7646                //tableModel如果非determined，请不要添加到sqlenv里
7647                if (sqlenv != null && tableColumn!=null) {
7648                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
7649                        if (schema != null) {
7650                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
7651                                if (tableColumn.hasStarLinkColumn()) {
7652                                        for (String column : tableColumn.getStarLinkColumnNames()) {
7653                                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(column));
7654                                        }
7655                                } else {
7656                                        tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getName()));
7657                                }
7658                        }
7659                }
7660        }
7661
7662        private void appendTableColumnToSQLEnv(Table tableModel, TObjectName tableColumn) {
7663                if (sqlenv != null) {
7664                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
7665                        if (schema != null) {
7666                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
7667                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getColumnNameOnly()));
7668                        }
7669                }
7670        }
7671
7672        private void analyzeCreateStageStmt(TCreateStageStmt stmt) {
7673                TObjectName stageName = stmt.getStageName();
7674
7675                if (stageName != null) {
7676
7677                        Table tableModel = modelFactory.createStage(stageName);
7678                        tableModel.setCreateTable(true);
7679                        tableModel.setStage(true);
7680                        tableModel.setLocation(stmt.getExternalStageURL());
7681
7682                        Process process = modelFactory.createProcess(stmt);
7683                        tableModel.addProcess(process);
7684
7685                        TableColumn locationColumn = null;
7686                        if (stmt.getExternalStageURL() != null) {
7687                                TObjectName location = new TObjectName();
7688                                location.setString(stmt.getExternalStageURL());
7689                                locationColumn = modelFactory.createStageLocation(tableModel, location);
7690
7691                                Table pathModel = modelFactory.createTableByName(stmt.getExternalStageURL(), true);
7692                                pathModel.setPath(true);
7693                                pathModel.setCreateTable(true);
7694                                TObjectName fileUri = new TObjectName();
7695                                fileUri.setString(stmt.getExternalStageURL());
7696                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
7697
7698                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7699                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7700                                relation.setTarget(new TableColumnRelationshipElement(locationColumn));
7701                                relation.setProcess(process);
7702
7703                        } else {
7704                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
7705                                        if (temp instanceof TPutStmt) {
7706                                                TPutStmt put = (TPutStmt) temp;
7707                                                TStageLocation stageLocation = put.getStageLocation();
7708                                                if (stageLocation != null && stageLocation.getStageName() != null) {
7709                                                        Table stage = modelManager.getTableByName(
7710                                                                        DlineageUtil.getTableFullName(stageLocation.getStageName().toString()));
7711                                                        if (stage == tableModel) {
7712                                                                TObjectName location = new TObjectName();
7713                                                                if (!SQLUtil.isEmpty(put.getFileName())) {
7714                                                                        location.setString(put.getFileName());
7715                                                                        locationColumn = modelFactory.createStageLocation(tableModel, location);
7716                                                                }
7717                                                                break;
7718                                                        }
7719                                                }
7720                                        }
7721                                }
7722                        }
7723
7724                        String fileFormat = stmt.getFileFormatName();
7725                        if (fileFormat != null) {
7726                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
7727                                        if (temp instanceof TCreateFileFormatStmt) {
7728                                                TCreateFileFormatStmt fileFormatStmt = (TCreateFileFormatStmt) temp;
7729                                                if (fileFormatStmt.getFileFormatName() != null
7730                                                                && fileFormatStmt.getFileFormatName().toString().equalsIgnoreCase(fileFormat)) {
7731                                                        tableModel.setFileType(fileFormatStmt.getTypeName());
7732                                                        break;
7733                                                }
7734                                        }
7735                                }
7736                        }
7737
7738                        String procedureParent = getProcedureParentName(stmt);
7739                        if (procedureParent != null) {
7740                                tableModel.setParent(procedureParent);
7741                        }
7742
7743                        if (locationColumn != null) {
7744                                List<Table> tables = modelManager.getTablesByName();
7745                                if (tables != null) {
7746                                        for (Table referTable : tables) {
7747                                                if (referTable.getLocation() != null && referTable.getLocation()
7748                                                                .equals(DlineageUtil.getIdentifierNormalTableName(tableModel.getName()))) {
7749                                                        for (int i = 0; i < referTable.getColumns().size(); i++) {
7750                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7751                                                                relation.addSource(new TableColumnRelationshipElement(locationColumn));
7752                                                                relation.setTarget(new TableColumnRelationshipElement(referTable.getColumns().get(i)));
7753                                                        }
7754                                                }
7755                                        }
7756                                }
7757                        }
7758                } else {
7759                        ErrorInfo errorInfo = new ErrorInfo();
7760                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7761                        errorInfo.setErrorMessage("Can't get target table. CreateStageStmt is " + stmt.toString());
7762                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
7763                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
7764                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
7765                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
7766                                        ModelBindingManager.getGlobalHash()));
7767                        errorInfo.fillInfo(this);
7768                        errorInfos.add(errorInfo);
7769                }
7770        }
7771
7772        private void analyzeCreateExternalDataSourceStmt(TCreateExternalDataSourceStmt stmt) {
7773                TObjectName dataSourceName = stmt.getDataSourceName();
7774                String locationUrl = stmt.getOption("LOCATION");
7775                if (dataSourceName != null && locationUrl != null) {
7776                        Table tableModel = modelFactory.createDataSource(dataSourceName);
7777                        tableModel.setCreateTable(true);
7778                        tableModel.setDataSource(true);
7779                        tableModel.setLocation(locationUrl);
7780
7781                        Process process = modelFactory.createProcess(stmt);
7782                        tableModel.addProcess(process);
7783
7784                        TObjectName location = new TObjectName();
7785                        location.setString(locationUrl);
7786                        modelFactory.createTableColumn(tableModel, location, true);
7787
7788                        String procedureParent = getProcedureParentName(stmt);
7789                        if (procedureParent != null) {
7790                                tableModel.setParent(procedureParent);
7791                        }
7792                } else {
7793                        ErrorInfo errorInfo = new ErrorInfo();
7794                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7795                        errorInfo.setErrorMessage("Can't get target table. CreateExternalDataSourceStmt is " + stmt.toString());
7796                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
7797                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
7798                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
7799                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
7800                                        ModelBindingManager.getGlobalHash()));
7801                        errorInfo.fillInfo(this);
7802                        errorInfos.add(errorInfo);
7803                }
7804        }
7805
7806        private void analyzeCreateStreamStmt(TCreateStreamStmt stmt) {
7807                TObjectName streamName = stmt.getStreamName();
7808
7809                if (streamName != null) {
7810                        Table tableModel = modelFactory.createTableByName(stmt.getTableName());
7811                        tableModel.setCreateTable(true);
7812
7813                        Table streamModel = modelFactory.createStream(streamName);
7814                        streamModel.setCreateTable(true);
7815                        streamModel.setStream(true);
7816
7817                        Process process = modelFactory.createProcess(stmt);
7818                        tableModel.addProcess(process);
7819
7820                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7821                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(streamModel.getRelationRows()));
7822                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7823                        relation.setProcess(process);
7824                }
7825        }
7826
7827        private String getProcedureParentName(TCustomSqlStatement stmt) {
7828                if (stmt instanceof TStoredProcedureSqlStatement) {
7829                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
7830                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
7831                        }
7832                }
7833
7834                stmt = stmt.getParentStmt();
7835                if (stmt == null)
7836                        return null;
7837
7838        if (stmt instanceof TCommonBlock) {
7839                        if(((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
7840                                stmt = (TStoredProcedureSqlStatement)((TCommonBlock) stmt).getBlockBody().getParentObjectName();
7841                        }
7842                }
7843
7844                if (stmt instanceof TStoredProcedureSqlStatement) {
7845                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
7846                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
7847                        }
7848                }
7849                if (stmt instanceof TTeradataCreateProcedure) {
7850                        if (((TTeradataCreateProcedure) stmt).getProcedureName() != null) {
7851                                return ((TTeradataCreateProcedure) stmt).getProcedureName().toString();
7852                        }
7853                }
7854
7855                return getProcedureParentName(stmt);
7856        }
7857
7858        private TStoredProcedureSqlStatement getProcedureParent(TCustomSqlStatement stmt) {
7859                if (stmt instanceof TStoredProcedureSqlStatement) {
7860                        return (TStoredProcedureSqlStatement) stmt;
7861                }
7862
7863                stmt = stmt.getParentStmt();
7864                if (stmt == null)
7865                        return null;
7866
7867                if (stmt instanceof TCommonBlock) {
7868                        if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
7869                                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
7870                        }
7871                }
7872
7873                if (stmt instanceof TStoredProcedureSqlStatement) {
7874                        return ((TStoredProcedureSqlStatement) stmt);
7875                }
7876                if (stmt instanceof TTeradataCreateProcedure) {
7877                        return ((TTeradataCreateProcedure) stmt);
7878                }
7879
7880                return getProcedureParent(stmt);
7881        }
7882
7883        private void analyzeMergeStmt(TMergeSqlStatement stmt) {
7884                Object tableModel;
7885                Process process;
7886                if (stmt.getUsingTable() != null) {
7887                        TTable table = stmt.getTargetTable();
7888                        if(table.getSubquery()!=null) {
7889                                tableModel = modelFactory.createQueryTable(table);
7890                                analyzeSelectStmt(table.getSubquery());
7891                                process = modelFactory.createProcess(stmt);
7892                        }
7893                        else {
7894                                tableModel = modelFactory.createTable(table);
7895                                process = modelFactory.createProcess(stmt);
7896                                ((Table)tableModel).addProcess(process);
7897                        }
7898                        
7899                        for(TTable item: stmt.tables) {
7900                                if(item.getSubquery()!=null) {
7901                                        continue;
7902                                }
7903                                Table tableItemModel = modelFactory.createTable(item);                          
7904                                if (tableItemModel.getColumns() == null || tableItemModel.getColumns().isEmpty()) {
7905                                        tableItemModel.addColumnsFromSQLEnv();
7906                                }
7907                        }
7908
7909                        if (stmt.getUsingTable().getSubquery() != null) {
7910                                QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable());
7911                                analyzeSelectStmt(stmt.getUsingTable().getSubquery());
7912
7913                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getUsingTable().getSubquery());
7914                                
7915                                if (queryTable != null && resultSetModel != null && queryTable != resultSetModel) {
7916                                        if (queryTable.getColumns().size() == resultSetModel.getColumns().size()) {
7917                                                for (int i = 0; i < queryTable.getColumns().size(); i++) {
7918                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7919                                                        relation.setEffectType(EffectType.select);
7920                                                        relation.setTarget(new ResultColumnRelationshipElement(queryTable.getColumns().get(i)));
7921                                                        relation.addSource(new ResultColumnRelationshipElement(resultSetModel.getColumns().get(i)));
7922                                                        relation.setProcess(process);
7923                                                }
7924                                        }
7925                                }
7926                                
7927                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
7928                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
7929                                        impactRelation.setEffectType(EffectType.merge);
7930                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
7931                                                        resultSetModel.getRelationRows()));
7932                                        if (tableModel instanceof Table) {
7933                                                impactRelation.setTarget(
7934                                                                new RelationRowsRelationshipElement<TableRelationRows>(((Table)tableModel).getRelationRows()));
7935                                        }
7936                                        else {
7937                                                impactRelation.setTarget(
7938                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)tableModel).getRelationRows()));
7939                                        }
7940                                }
7941                        } else {
7942                                if (stmt.getUsingTable().getAliasClause() != null && stmt.getUsingTable().getAliasClause().getColumns() != null && stmt.getUsingTable().getValueClause().getRows()!=null) {
7943                                        Table usingTable = modelFactory.createTableFromCreateDDL(stmt.getUsingTable(), false, stmt.getUsingTable().getAliasName() + stmt.getUsingTable().getTableName());
7944                                        usingTable.setCreateTable(true);
7945                                        usingTable.setSubType(SubType.function);
7946                                        for (int z=0;z<stmt.getUsingTable().getAliasClause().getColumns().size();z++) {
7947                                                TObjectName columnName = stmt.getUsingTable().getAliasClause().getColumns().getObjectName(z);
7948                                                TableColumn tableColumn = modelFactory.createTableColumn(usingTable, columnName, true);
7949                                                TResultColumn resultColumn = stmt.getUsingTable().getValueClause().getRows().get(0).getResultColumn(z);
7950                                                modelManager.bindModel(resultColumn, tableColumn);
7951                                                analyzeResultColumnExpressionRelation(tableColumn, resultColumn.getExpr());                                             }
7952                                }
7953                                else {
7954                                        modelFactory.createTable(stmt.getUsingTable());
7955                                }
7956                        }
7957                
7958
7959                        if (stmt.getWhenClauses() != null && stmt.getWhenClauses().size() > 0) {
7960                                for (int i = 0; i < stmt.getWhenClauses().size(); i++) {
7961                                        TMergeWhenClause clause = stmt.getWhenClauses().getElement(i);
7962                                        if (clause.getCondition() != null) {
7963                                                analyzeFilterCondition(null, clause.getCondition(), null, null, EffectType.merge_when);
7964                                        }
7965                                        if (clause.getUpdateClause() != null) {
7966                                                TResultColumnList columns = clause.getUpdateClause().getUpdateColumnList();
7967                                                if (columns == null || columns.size() == 0)
7968                                                        continue;
7969
7970                                                ResultSet resultSet = modelFactory.createResultSet(clause.getUpdateClause(), false);
7971                                                createPseudoImpactRelation(stmt, resultSet, EffectType.merge_update);
7972
7973                                                for (int j = 0; j < columns.size(); j++) {
7974                                                        TResultColumn resultColumn = columns.getResultColumn(j);
7975                                                        if (resultColumn.getExpr().getLeftOperand()
7976                                                                        .getExpressionType() == EExpressionType.simple_object_name_t) {
7977                                                                TObjectName columnObject = resultColumn.getExpr().getLeftOperand().getObjectOperand();
7978
7979                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
7980                                                                        continue;
7981                                                                }
7982
7983                                                                if (columnObject.getColumnNameOnly().startsWith("@")
7984                                                                                && (option.getVendor() == EDbVendor.dbvmssql
7985                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
7986                                                                        continue;
7987                                                                }
7988
7989                                                                if (columnObject.getColumnNameOnly().startsWith(":")
7990                                                                                && (option.getVendor() == EDbVendor.dbvhana
7991                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
7992                                                                        continue;
7993                                                                }
7994
7995                                                                ResultColumn updateColumn = modelFactory.createMergeResultColumn(resultSet,
7996                                                                                columnObject);
7997
7998                                                                TExpression valueExpression = resultColumn.getExpr().getRightOperand();
7999                                                                if (valueExpression == null)
8000                                                                        continue;
8001
8002                                                                columnsInExpr visitor = new columnsInExpr();
8003                                                                valueExpression.inOrderTraverse(visitor);
8004                                                                List<TObjectName> objectNames = visitor.getObjectNames();
8005                                                                List<TParseTreeNode> functions = visitor.getFunctions();
8006
8007                                                                if (functions != null && !functions.isEmpty()) {
8008                                                                        analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.merge_update);
8009                                                                }
8010
8011                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8012                                                                if (subquerys != null && !subquerys.isEmpty()) {
8013                                                                        analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.merge_update);
8014                                                                }
8015
8016                                                                analyzeDataFlowRelation(updateColumn, objectNames, EffectType.merge_update, functions);
8017
8018                                                                List<TParseTreeNode> constants = visitor.getConstants();
8019                                                                analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.merge_update,
8020                                                                                functions);
8021
8022                                                                if (tableModel instanceof Table) {
8023                                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)tableModel, columnObject,
8024                                                                                        false);
8025
8026                                                                        if (tableColumn != null) {
8027                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8028                                                                                relation.setEffectType(EffectType.merge_update);
8029                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8030                                                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
8031                                                                                relation.setProcess(process);
8032                                                                        }
8033                                                                }
8034                                                                else {
8035                                                                        TTable targetTable = stmt.getTargetTable().getSubquery().getTables().getTable(0);
8036                                                                        if (targetTable != null && modelManager.getModel(targetTable) instanceof Table) {
8037                                                                                Table targetTableModel = (Table) modelManager.getModel(targetTable);
8038                                                                                TableColumn tableColumn = modelFactory
8039                                                                                                .createTableColumn((Table) targetTableModel, columnObject, false);
8040                                                                                if (tableColumn != null) {
8041                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8042                                                                                        relation.setEffectType(EffectType.merge_update);
8043                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8044                                                                                        relation.addSource(new ResultColumnRelationshipElement(updateColumn));
8045                                                                                        relation.setProcess(process);
8046                                                                                }
8047                                                                        }
8048                                                                }
8049                                                        }
8050                                                }
8051                                        }
8052                                        if (clause.getInsertClause() != null && tableModel instanceof Table) {
8053                                                TExpression insertValue = clause.getInsertClause().getInsertValue();
8054                                                if (insertValue != null
8055                                                                && insertValue.getExpressionType() == EExpressionType.objectConstruct_t) {
8056                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
8057
8058                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
8059
8060                                                        TObjectConstruct objectConstruct = insertValue.getObjectConstruct();
8061                                                        for (int z = 0; z < objectConstruct.getPairs().size(); z++) {
8062                                                                TPair pair = objectConstruct.getPairs().getElement(z);
8063
8064                                                                if (pair.getKeyName().getExpressionType() == EExpressionType.simple_constant_t) {
8065                                                                        TObjectName columnObject = new TObjectName();
8066                                                                        TConstant constant = pair.getKeyName().getConstantOperand();
8067                                                                        TSourceToken newSt = new TSourceToken(
8068                                                                                        constant.getValueToken().getTextWithoutQuoted());
8069                                                                        columnObject.setPartToken(newSt);
8070                                                                        columnObject.setSourceTable(stmt.getTargetTable());
8071                                                                        columnObject.setStartToken(constant.getStartToken());
8072                                                                        columnObject.setEndToken(constant.getEndToken());
8073
8074                                                                        ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
8075                                                                                        columnObject);
8076
8077                                                                        TExpression valueExpression = pair.getKeyValue();
8078                                                                        if (valueExpression == null)
8079                                                                                continue;
8080
8081                                                                        columnsInExpr visitor = new columnsInExpr();
8082                                                                        valueExpression.inOrderTraverse(visitor);
8083                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
8084                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
8085
8086                                                                        if (functions != null && !functions.isEmpty()) {
8087                                                                                analyzeFunctionDataFlowRelation(insertColumn, functions,
8088                                                                                                EffectType.merge_insert);
8089                                                                        }
8090
8091                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8092                                                                        if (subquerys != null && !subquerys.isEmpty()) {
8093                                                                                analyzeSubqueryDataFlowRelation(insertColumn, subquerys,
8094                                                                                                EffectType.merge_insert);
8095                                                                        }
8096
8097                                                                        analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert,
8098                                                                                        functions);
8099
8100                                                                        List<TParseTreeNode> constants = visitor.getConstants();
8101                                                                        analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
8102                                                                                        functions);
8103
8104                                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)tableModel,
8105                                                                                        columnObject, false);
8106
8107                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8108                                                                        relation.setEffectType(EffectType.merge_insert);
8109                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8110                                                                        relation.addSource(new ResultColumnRelationshipElement(insertColumn));
8111                                                                        relation.setProcess(process);
8112                                                                }
8113                                                        }
8114                                                } else {
8115                                                        TObjectNameList columns = clause.getInsertClause().getColumnList();
8116                                                        TResultColumnList values = clause.getInsertClause().getValuelist();
8117                                                        if (values == null || values.size() == 0) {
8118                                                                if (clause.getInsertClause().toString().toLowerCase().indexOf("row") != -1) {
8119                                                                        if (stmt.getUsingTable().getSubquery() != null) {
8120                                                                                ResultSet sourceResultSet = modelFactory.createQueryTable(stmt.getUsingTable());
8121                                                                                TObjectName targetStarColumn = new TObjectName();
8122                                                                                targetStarColumn.setString("*");
8123                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn((Table)tableModel,
8124                                                                                                targetStarColumn, true);
8125                                                                                if (sourceResultSet.getColumns() == null
8126                                                                                                || sourceResultSet.getColumns().isEmpty()) {
8127                                                                                        TObjectName sourceStarColumn = new TObjectName();
8128                                                                                        sourceStarColumn.setString("*");
8129                                                                                        modelFactory.createResultColumn(sourceResultSet, sourceStarColumn);
8130                                                                                }
8131                                                                                for (ResultColumn sourceColumn : sourceResultSet.getColumns()) {
8132                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8133                                                                                        relation.setEffectType(EffectType.merge_insert);
8134                                                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
8135                                                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
8136                                                                                        relation.setProcess(process);
8137                                                                                }
8138                                                                        } else {
8139                                                                                Table sourceTable = modelFactory.createTable(stmt.getUsingTable());
8140                                                                                TObjectName sourceStarColumn = new TObjectName();
8141                                                                                sourceStarColumn.setString("*");
8142                                                                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable,
8143                                                                                                sourceStarColumn, true);
8144                                                                                TObjectName targetStarColumn = new TObjectName();
8145                                                                                targetStarColumn.setString("*");
8146                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn(((Table)tableModel),
8147                                                                                                targetStarColumn, true);
8148                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8149                                                                                relation.setEffectType(EffectType.merge_insert);
8150                                                                                relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
8151                                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
8152                                                                                relation.setProcess(process);
8153                                                                        }
8154
8155                                                                }
8156                                                                continue;
8157                                                        }
8158
8159                                                        List<TObjectName> tableColumns = new ArrayList<TObjectName>();
8160                                                        if (columns == null || columns.size() == 0) {
8161//                                                              if (!((Table)tableModel).getColumns().isEmpty()) {
8162//                                                                      for (int j = 0; j < ((Table)tableModel).getColumns().size(); j++) {
8163//                                                                              if (((Table)tableModel).getColumns().get(j).getColumnObject() == null) {
8164//                                                                                      continue;
8165//                                                                              }
8166//                                                                              tableColumns.add(((Table)tableModel).getColumns().get(j).getColumnObject());
8167//                                                                      }
8168//                                                              } else {
8169                                                                        for (int j = 0; j < values.size(); j++) {
8170                                                                                TResultColumn column = values.getResultColumn(j);
8171                                                                                if (column.getAliasClause() != null) {
8172                                                                                        tableColumns.add(column.getAliasClause().getAliasName());
8173                                                                                } else if (column.getFieldAttr() != null) {
8174                                                                                        tableColumns.add(column.getFieldAttr());
8175                                                                                } else {
8176                                                                                        TObjectName columnName = new TObjectName();
8177                                                                                        columnName.setString(column.toString());
8178                                                                                        tableColumns.add(columnName);
8179                                                                                }
8180//                                                                      }
8181                                                                }
8182                                                        } else {
8183                                                                for (int j = 0; j < columns.size(); j++) {
8184                                                                        tableColumns.add(columns.getObjectName(j));
8185                                                                }
8186                                                        }
8187
8188                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
8189
8190                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
8191
8192                                                        for (int j = 0; j < tableColumns.size() && j < values.size(); j++) {
8193                                                                TObjectName columnObject = tableColumns.get(j);
8194
8195                                                                ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
8196                                                                                columnObject);
8197
8198                                                                TExpression valueExpression = values.getResultColumn(j).getExpr();
8199                                                                if (valueExpression == null)
8200                                                                        continue;
8201
8202                                                                columnsInExpr visitor = new columnsInExpr();
8203                                                                valueExpression.inOrderTraverse(visitor);
8204                                                                List<TObjectName> objectNames = visitor.getObjectNames();
8205                                                                List<TParseTreeNode> functions = visitor.getFunctions();
8206
8207                                                                if (functions != null && !functions.isEmpty()) {
8208                                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, EffectType.merge_insert);
8209                                                                }
8210
8211                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8212                                                                if (subquerys != null && !subquerys.isEmpty()) {
8213                                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, EffectType.merge_insert);
8214                                                                }
8215
8216                                                                analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert, functions);
8217
8218                                                                List<TParseTreeNode> constants = visitor.getConstants();
8219                                                                analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
8220                                                                                functions);
8221
8222                                                                TableColumn tableColumn = modelFactory.createTableColumn(((Table)tableModel), columnObject,
8223                                                                                false);
8224                                                                if(tableColumn == null) {
8225                                                                        if (((Table) tableModel).isCreateTable()) {
8226                                                                                tableColumn = ((Table) tableModel).getColumns().get(j);
8227                                                                        }
8228                                                                        else {
8229                                                                                continue;
8230                                                                        }
8231                                                                }
8232
8233                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8234                                                                relation.setEffectType(EffectType.merge_insert);
8235                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8236                                                                relation.addSource(new ResultColumnRelationshipElement(insertColumn));
8237                                                                relation.setProcess(process);
8238                                                        }
8239                                                }
8240                                        }
8241                                }
8242
8243                        }
8244
8245                        if (stmt.getCondition() != null) {
8246                                analyzeFilterCondition(null, stmt.getCondition(), null, JoinClauseType.on, EffectType.merge);
8247                        }
8248                }
8249        }
8250
8251        private List<TableColumn> bindInsertTableColumn(Table tableModel, TInsertIntoValue value, List<TObjectName> keyMap,
8252                        List<TResultColumn> valueMap) {
8253                List<TableColumn> tableColumns = new ArrayList<TableColumn>();
8254                if (value.getColumnList() != null) {
8255                        for (int z = 0; z < value.getColumnList().size(); z++) {
8256                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
8257                                                value.getColumnList().getObjectName(z));
8258                                tableColumns.add(tableColumn);
8259                                keyMap.add(tableColumn.getColumnObject());
8260                        }
8261                }
8262
8263                if (value.getTargetList() != null) {
8264                        for (int z = 0; z < value.getTargetList().size(); z++) {
8265                                TMultiTarget target = value.getTargetList().getMultiTarget(z);
8266                                TResultColumnList columns = target.getColumnList();
8267                                for (int i = 0; i < columns.size(); i++) {
8268                                        if (value.getColumnList() == null) {
8269                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
8270                                                                columns.getResultColumn(i).getFieldAttr());
8271                                                tableColumns.add(tableColumn);
8272                                        }
8273                                        valueMap.add(columns.getResultColumn(i));
8274                                }
8275                        }
8276                }
8277
8278                return tableColumns;
8279        }
8280
8281        private TableColumn matchColumn(List<TableColumn> tableColumns, TableColumn targetColumn) {
8282                String columnName = targetColumn.getName();
8283                if (tableColumns == null) {
8284                        return null;
8285                }
8286                for (int i = 0; i < tableColumns.size(); i++) {
8287                        TableColumn column = tableColumns.get(i);
8288                        if (column.getColumnObject() == null) {
8289                                continue;
8290                        }
8291                        if(column.isStruct() && targetColumn.isStruct()) {
8292                                List<String> names = SQLUtil.parseNames(column.getName());
8293                                List<String> targetNames = SQLUtil
8294                                                .parseNames(targetColumn.getName());
8295                                if (!getColumnName(targetNames.get(0))
8296                                                .equals(getColumnName(names.get(0)))) {
8297                                        continue;
8298                                }
8299                        }
8300                        if (getColumnName(column.getColumnObject().toString()).equals(getColumnName(columnName))) {
8301                                return column;
8302                        }
8303                }
8304                return null;
8305        }
8306        
8307        private TableColumn matchColumn(List<TableColumn> tableColumns, TObjectName columnName) {
8308                if (tableColumns == null) {
8309                        return null;
8310                }
8311                for (int i = 0; i < tableColumns.size(); i++) {
8312                        TableColumn column = tableColumns.get(i);
8313                        if (column.getColumnObject() == null) {
8314                                continue;
8315                        }
8316                        if (DlineageUtil.getColumnName(column.getColumnObject()).equalsIgnoreCase(DlineageUtil.getColumnName(columnName)))
8317                                return column;
8318                }
8319                return null;
8320        }
8321
8322        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, ResultColumn resultColumn) {
8323                if (resultColumns == null) {
8324                        return null;
8325                }
8326
8327                TObjectName columnName = getObjectName(resultColumn);
8328                if (columnName == null) {
8329                        return null;
8330                }
8331
8332                for (int i = 0; i < resultColumns.size(); i++) {
8333                        ResultColumn column = resultColumns.get(i);
8334                        if (column.getAlias() != null
8335                                        && getColumnName(column.getAlias()).equalsIgnoreCase(getColumnName(columnName)))
8336                                return column;
8337                        if (column.getName() != null && getColumnName(column.getName()).equalsIgnoreCase(getColumnName(columnName)))
8338                                return column;
8339                        if (column.getName() != null && column.getName().endsWith("*")) {
8340                                if ("*".equals(column.getColumnObject().toString())) {
8341                                        return column;
8342                                } else {
8343                                        TObjectName columnObjectName = getObjectName(column);
8344                                        if (columnObjectName.getTableString() != null
8345                                                        && columnObjectName.getTableString().equals(getResultSetAlias(resultColumn))) {
8346                                                return column;
8347                                        }
8348                                }
8349                        }
8350                }
8351                return null;
8352        }
8353
8354        private String getResultSetAlias(ResultColumn resultColumn) {
8355                ResultSet resultSet = resultColumn.getResultSet();
8356                if (resultSet instanceof QueryTable) {
8357                        return ((QueryTable) resultSet).getAlias();
8358                }
8359                return null;
8360        }
8361
8362        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, TObjectName columnName) {
8363                if (resultColumns == null) {
8364                        return null;
8365                }
8366                for (int i = 0; i < resultColumns.size(); i++) {
8367                        ResultColumn column = resultColumns.get(i);
8368                        if (column.getAlias() != null
8369                                        && getColumnName(column.getAlias()).equalsIgnoreCase(getColumnName(columnName)))
8370                                return column;
8371                        if (column.getName() != null && getColumnName(column.getName()).equalsIgnoreCase(getColumnName(columnName)))
8372                                return column;
8373                        if (column.getName() != null && column.getName().endsWith("*")) {
8374                                if ("*".equals(column.getColumnObject().toString())) {
8375                                        return column;
8376                                } else {
8377                                        TObjectName columnObjectName = getObjectName(column);
8378                                        if (columnObjectName.getTableString() != null
8379                                                        && columnObjectName.getTableString().equals(columnName.getTableString())) {
8380                                                return column;
8381                                        }
8382                                }
8383                        }
8384                }
8385                return null;
8386        }
8387
8388        private void analyzeInsertStmt(TInsertSqlStatement stmt) {
8389                Map<Table, List<TObjectName>> insertTableKeyMap = new LinkedHashMap<Table, List<TObjectName>>();
8390                Map<Table, List<TResultColumn>> insertTableValueMap = new LinkedHashMap<Table, List<TResultColumn>>();
8391                Map<String, List<TableColumn>> tableColumnMap = new LinkedHashMap<String, List<TableColumn>>();
8392                List<Table> inserTables = new ArrayList<Table>();
8393                List<TExpression> expressions = new ArrayList<TExpression>();
8394                boolean hasInsertColumns = false;
8395                
8396                EffectType effectType = EffectType.insert;
8397                if(stmt.getInsertToken()!=null && stmt.getInsertToken().toString().toLowerCase().startsWith("replace")) {
8398                        effectType = EffectType.replace;
8399                }
8400                
8401                if (stmt.getInsertConditions() != null && stmt.getInsertConditions().size() > 0) {
8402                        for (int i = 0; i < stmt.getInsertConditions().size(); i++) {
8403                                TInsertCondition condition = stmt.getInsertConditions().getElement(i);
8404                                if (condition.getCondition() != null) {
8405                                        expressions.add(condition.getCondition());
8406                                }
8407                                for (int j = 0; j < condition.getInsertIntoValues().size(); j++) {
8408                                        TInsertIntoValue value = condition.getInsertIntoValues().getElement(j);
8409                                        TTable table = value.getTable();
8410                                        Table tableModel = modelFactory.createTable(table);
8411
8412                                        inserTables.add(tableModel);
8413                                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
8414                                        List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
8415                                        insertTableKeyMap.put(tableModel, keyMap);
8416                                        insertTableValueMap.put(tableModel, valueMap);
8417
8418                                        List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
8419                                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null
8420                                                        && !tableColumns.isEmpty()) {
8421                                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
8422                                                                tableColumns);
8423                                        }
8424
8425                                        // if (stmt.getSubQuery() != null)
8426                                        {
8427                                                Process process = modelFactory.createProcess(stmt);
8428                                                tableModel.addProcess(process);
8429                                        }
8430                                }
8431                        }
8432                        hasInsertColumns = true;
8433                } else if (stmt.getInsertIntoValues() != null && stmt.getInsertIntoValues().size() > 0) {
8434                        for (int i = 0; i < stmt.getInsertIntoValues().size(); i++) {
8435                                TInsertIntoValue value = stmt.getInsertIntoValues().getElement(i);
8436                                TTable table = value.getTable();
8437                                Table tableModel = modelFactory.createTable(table);
8438
8439                                inserTables.add(tableModel);
8440                                List<TObjectName> keyMap = new ArrayList<TObjectName>();
8441                                List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
8442                                insertTableKeyMap.put(tableModel, keyMap);
8443                                insertTableValueMap.put(tableModel, valueMap);
8444
8445                                List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
8446                                if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
8447                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
8448                                }
8449
8450                                // if (stmt.getSubQuery() != null)
8451                                {
8452                                        Process process = modelFactory.createProcess(stmt);
8453                                        tableModel.addProcess(process);
8454                                }
8455                        }
8456                        hasInsertColumns = true;
8457                } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
8458                        TTable table = stmt.getTargetTable();
8459                        Table tableModel = modelFactory.createTable(table);
8460
8461                        inserTables.add(tableModel);
8462                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
8463                        insertTableKeyMap.put(tableModel, keyMap);
8464                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
8465                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
8466                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
8467                                                stmt.getColumnList().getObjectName(i));
8468                                tableColumns.add(tableColumn);
8469                        }
8470                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
8471                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
8472                        }
8473
8474                        // if (stmt.getSubQuery() != null)
8475                        {
8476                                Process process = modelFactory.createProcess(stmt);
8477                                tableModel.addProcess(process);
8478                        }
8479                        hasInsertColumns = true;
8480                } else if (stmt.getOutputClause() != null && stmt.getOutputClause().getSelectItemList().size() > 0) {
8481                        TTable table = stmt.getTargetTable();
8482                        Table tableModel = modelFactory.createTable(table);
8483
8484                        inserTables.add(tableModel);
8485                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
8486                        insertTableKeyMap.put(tableModel, keyMap);
8487                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
8488                        for (int i = 0; i < stmt.getOutputClause().getSelectItemList().size(); i++) {
8489                                TObjectName columnName = stmt.getOutputClause().getSelectItemList().getResultColumn(i).getFieldAttr();
8490                                if (columnName.getPseudoTableType() != EPseudoTableType.none) {
8491                                        // Phase 1 already swapped tokens; getColumnNameOnly() returns actual column name
8492                                        String column = columnName.getColumnNameOnly();
8493                                        columnName = new TObjectName();
8494                                        columnName.setString(column);
8495                                } else {
8496                                        String column = columnName.toString().toLowerCase();
8497                                        if ((column.startsWith("inserted.") || column.startsWith("deleted."))
8498                                                        && columnName.getPropertyToken() != null) {
8499                                                column = columnName.getPropertyToken().getAstext();
8500                                                columnName = new TObjectName();
8501                                                columnName.setString(column);
8502                                        }
8503                                }
8504                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, columnName);
8505                                tableColumns.add(tableColumn);
8506                        }
8507                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
8508                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
8509                        }
8510
8511                        // if (stmt.getSubQuery() != null)
8512                        {
8513                                Process process = modelFactory.createProcess(stmt);
8514                                tableModel.addProcess(process);
8515                        }
8516                        hasInsertColumns = true;
8517                } else {
8518                        TTable table = stmt.getTargetTable();
8519                        Table tableModel;
8520                        if (table != null) {
8521                                tableModel = modelFactory.createTable(table);
8522                                // if (stmt.getSubQuery() != null)
8523                                {
8524                                        Process process = modelFactory.createProcess(stmt);
8525                                        tableModel.addProcess(process);
8526                                }
8527                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
8528                                        tableModel.addColumnsFromSQLEnv();
8529                                }
8530                        } else if (stmt.getDirectoryName() != null) {
8531                                tableModel = modelFactory.createTableByName(stmt.getDirectoryName(), true);
8532                                tableModel.setPath(true);
8533                                tableModel.setCreateTable(true);
8534                                TObjectName fileUri = new TObjectName();
8535                                fileUri.setString("uri=" + stmt.getDirectoryName());
8536                                TableColumn tableColumn = modelFactory.createFileUri(tableModel, fileUri);
8537                                // if (stmt.getSubQuery() != null)
8538                                {
8539                                        Process process = modelFactory.createProcess(stmt);
8540                                        tableModel.addProcess(process);
8541                                }
8542                        } else {
8543                                ErrorInfo errorInfo = new ErrorInfo();
8544                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
8545                                errorInfo.setErrorMessage("Can't get target table. InsertSqlStatement is " + stmt.toString());
8546                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
8547                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
8548                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
8549                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
8550                                                ModelBindingManager.getGlobalHash()));
8551                                errorInfo.fillInfo(this);
8552                                errorInfos.add(errorInfo);
8553                                return;
8554                        }
8555                        inserTables.add(tableModel);
8556                        if (table != null
8557                                        && tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null) {
8558                                if (tableModel.getColumns() != null && !tableModel.getColumns().isEmpty()) {
8559                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
8560                                                        tableModel.getColumns());
8561                                } else {
8562                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), null);
8563                                }
8564                        }
8565                }
8566
8567                if (stmt.getSubQuery() != null) {
8568                        analyzeSelectStmt(stmt.getSubQuery());
8569                }
8570
8571                Iterator<Table> tableIter = inserTables.iterator();
8572                while (tableIter.hasNext()) {
8573                        Table tableModel = tableIter.next();
8574                        List<TableColumn> tableColumns = tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()));
8575                        List<TObjectName> keyMap = insertTableKeyMap.get(tableModel);
8576                        List<TResultColumn> valueMap = insertTableValueMap.get(tableModel);
8577                        boolean initColumn = (hasInsertColumns && tableColumns != null && !containStarColumn(tableColumns));
8578
8579                        if (stmt.getSubQuery() != null) {
8580
8581                                List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
8582                                if (stmt.getSubQuery().getResultColumnList() != null || stmt.getSubQuery().getTransformClause() != null) {
8583                                        subquerys.add(stmt.getSubQuery());
8584                                } else if (stmt.getSubQuery().getValueClause() != null
8585                                                && stmt.getSubQuery().getValueClause().getRows() != null) {
8586                                        for (TResultColumnList resultColumnList : stmt.getSubQuery().getValueClause().getRows()) {
8587                                                for(TResultColumn resultColumn: resultColumnList) {
8588                                                        if(resultColumn.getExpr()!=null && resultColumn.getExpr().getSubQuery()!=null) {
8589                                                                analyzeSelectStmt(resultColumn.getExpr().getSubQuery());
8590                                                                subquerys.add(resultColumn.getExpr().getSubQuery());
8591                                                        }
8592                                                }
8593                                        }
8594                                }
8595
8596                                for(TSelectSqlStatement subquery: subquerys) {
8597                                        if ((tableModel.isCreateTable() && tableModel.getColumns() != null)
8598                                                        || (subquery.getSetOperatorType() == ESetOperatorType.none
8599                                                                        && stmt.getColumnList() != null && stmt.getColumnList().size() > 0)) {
8600
8601                                                ResultSet resultSetModel = null;
8602
8603                                                if (subquery != null) {
8604                                                        resultSetModel = (ResultSet) modelManager.getModel(subquery);
8605                                                }
8606
8607                                                TResultColumnList resultset = subquery.getResultColumnList();
8608                                                if (resultSetModel == null && resultset != null) {
8609                                                        resultSetModel = (ResultSet) modelManager.getModel(resultset);
8610                                                }
8611
8612                                                if (resultSetModel == null) {
8613                                                        ErrorInfo errorInfo = new ErrorInfo();
8614                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
8615                                                        errorInfo.setErrorMessage("Can't get resultset model");
8616                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(resultset.getStartToken().lineNo,
8617                                                                        resultset.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
8618                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(resultset.getEndToken().lineNo,
8619                                                                        resultset.getEndToken().columnNo + resultset.getEndToken().getAstext().length(),
8620                                                                        ModelBindingManager.getGlobalHash()));
8621                                                        errorInfos.add(errorInfo);
8622                                                }
8623
8624                                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
8625                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
8626                                                        impactRelation.setEffectType(effectType);
8627                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
8628                                                                        resultSetModel.getRelationRows()));
8629                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
8630                                                                        tableModel.getRelationRows()));
8631                                                }
8632
8633                                                int resultSetSize = resultSetModel.getColumns().size();
8634                                                int j = 0;
8635                                                int starIndex = 0;
8636                                                TObjectNameList items = stmt.getColumnList();
8637                                                List<String> itemNames = new ArrayList<String>();
8638                                                int starColumnCount = 0;
8639                                                for (ResultColumn item : resultSetModel.getColumns()) {
8640                                                        if (item.getName().endsWith("*")) {
8641                                                                starColumnCount += 1;
8642                                                        }
8643                                                }
8644                                                if (items != null) {
8645                                                        for (int i = 0; i < items.size() && j < resultSetSize; i++) {
8646                                                                TObjectName column = items.getObjectName(i);
8647
8648                                                                if (column.getDbObjectType() == EDbObjectType.variable) {
8649                                                                        continue;
8650                                                                }
8651
8652                                                                if (column.getColumnNameOnly().startsWith("@")
8653                                                                                && (option.getVendor() == EDbVendor.dbvmssql
8654                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
8655                                                                        continue;
8656                                                                }
8657
8658                                                                if (column.getColumnNameOnly().startsWith(":")
8659                                                                                && (option.getVendor() == EDbVendor.dbvhana
8660                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
8661                                                                        continue;
8662                                                                }
8663
8664                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
8665                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
8666                                                                        j++;
8667                                                                } else {
8668                                                                        starIndex++;
8669                                                                        if (resultSetSize - j == items.size() - i) {
8670                                                                                j++;
8671
8672                                                                        }
8673                                                                }
8674                                                                if (column != null) {
8675                                                                        TableColumn tableColumn;
8676                                                                        // if (!initColumn) {
8677                                                                        tableColumn = matchColumn(tableModel.getColumns(), column);
8678                                                                        if (tableColumn == null) {
8679                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
8680                                                                                        if (tableModel.getColumns().size() <= i) {
8681                                                                                                continue;
8682                                                                                        }
8683                                                                                        tableColumn = tableModel.getColumns().get(i);
8684                                                                                } else {
8685                                                                                        tableColumn = modelFactory.createTableColumn(tableModel, column, false);
8686                                                                                        if(tableColumn == null) {
8687                                                                                                continue;
8688                                                                                        }
8689                                                                                }
8690                                                                        }
8691//                                                      } else {
8692//                                                              tableColumn = matchColumn(tableColumns, column);
8693//                                                              if (tableColumn == null) {
8694//                                                                      continue;
8695//                                                              }
8696//                                                      }
8697                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8698                                                                        relation.setEffectType(effectType);
8699                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8700                                                                        if (resultColumn.hasStarLinkColumn()
8701                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1 && starColumnCount<=1) {
8702                                                                                boolean find = false;
8703                                                                                while (resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
8704                                                                                        TObjectName name = resultColumn.getStarLinkColumnName(starIndex - 1);
8705                                                                                        if (itemNames.contains(name.toString())) {
8706                                                                                                starIndex++;
8707                                                                                                continue;
8708                                                                                        }
8709                                                                                        ResultColumn expandStarColumn = modelFactory
8710                                                                                                        .createResultColumn(resultSetModel, name, false);
8711                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
8712                                                                                        itemNames.add(resultColumn.getName());
8713                                                                                        find = true;
8714                                                                                        break;
8715                                                                                }
8716                                                                                if (!find) {
8717                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8718                                                                                        itemNames.add(resultColumn.getName());
8719                                                                                }
8720                                                                        } else {
8721                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8722                                                                                itemNames.add(resultColumn.getName());
8723                                                                        }
8724                                                                        Process process = modelFactory.createProcess(stmt);
8725                                                                        relation.setProcess(process);
8726                                                                }
8727                                                        }
8728                                                } else {
8729                                                        List<TableColumn> columns = tableModel.getColumns();
8730                                                        if (columns.size() == 1 && tableModel.isPath()) {
8731                                                                for(int i=0;i<resultSetSize;i++) {
8732                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
8733                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8734                                                                        relation.setEffectType(effectType);
8735                                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(0)));
8736                                                                        if (resultColumn.hasStarLinkColumn()
8737                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
8738                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
8739                                                                                                resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
8740                                                                                                false);
8741                                                                                relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
8742                                                                        } else {
8743                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8744                                                                        }
8745                                                                        Process process = modelFactory.createProcess(stmt);
8746                                                                        relation.setProcess(process);
8747                                                                }
8748                                                        } else {
8749                                                                boolean fromStruct = false;
8750                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
8751                                                                        String column = columns.get(i).getName();
8752                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
8753                                                                        if (!resultColumn.getName().contains("*")) {
8754                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
8755                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
8756                                                                                                && resultSetSize == 1) {
8757                                                                                        starIndex++;
8758                                                                                        if (resultSetSize - j == columns.size() - i) {
8759                                                                                                j++;
8760
8761                                                                                        }
8762                                                                                }
8763                                                                                else {
8764                                                                                        j++;
8765                                                                                }
8766                                                                        } else {
8767                                                                                starIndex++;
8768                                                                                if (resultSetSize - j == columns.size() - i) {
8769                                                                                        j++;
8770
8771                                                                                }
8772                                                                        }
8773                                                                        if (column != null) {
8774                                                                                TableColumn tableColumn;
8775                                                                                // if (!initColumn) {
8776                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
8777                                                                                if (tableColumn == null) {
8778                                                                                        if (tableModel.isCreateTable()
8779                                                                                                        && !containStarColumn(tableModel.getColumns())) {
8780                                                                                                if (tableModel.getColumns().size() <= i) {
8781                                                                                                        continue;
8782                                                                                                }
8783                                                                                                tableColumn = tableModel.getColumns().get(i);
8784                                                                                        } else {
8785                                                                                                TObjectName columnName = new TObjectName();
8786                                                                                                columnName.setString(column);
8787                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
8788                                                                                                                false);
8789                                                                                        }
8790                                                                                }
8791                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
8792                                                                                        j--;
8793                                                                                        fromStruct = true;
8794                                                                                }
8795                                                                                if(fromStruct && !tableColumn.isStruct()) {
8796                                                                                        fromStruct = false;
8797                                                                                        resultColumn = resultSetModel.getColumns().get(j);
8798                                                                                        j++;
8799                                                                                }
8800                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8801                                                                                relation.setEffectType(effectType);
8802                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8803                                                                                if (resultColumn.hasStarLinkColumn()
8804                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
8805                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
8806                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
8807                                                                                                        false);
8808                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
8809                                                                                } else {
8810                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
8811                                                                                }
8812                                                                                Process process = modelFactory.createProcess(stmt);
8813                                                                                relation.setProcess(process);
8814                                                                        }
8815                                                                }
8816                                                        }
8817                                                }
8818                                        } else if (!subquery.isCombinedQuery()) {
8819                                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
8820                                                                .getModel(subquery.getResultColumnList() != null ? subquery.getResultColumnList()
8821                                                                                : subquery.getTransformClause());
8822
8823                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
8824                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
8825                                                        impactRelation.setEffectType(effectType);
8826                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
8827                                                                        resultSetModel.getRelationRows()));
8828                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
8829                                                                        tableModel.getRelationRows()));
8830                                                }
8831
8832                                                List<ResultColumn> columnsSnapshot = new ArrayList<ResultColumn>();
8833                                                columnsSnapshot.addAll(resultSetModel.getColumns());
8834                                                if(resultSetModel.isDetermined() && stmt.getColumnList() == null) {
8835                                                        tableModel.setDetermined(true);
8836                                                }
8837                                                for (int i = 0; i < columnsSnapshot.size(); i++) {
8838                                                        ResultColumn resultColumn = columnsSnapshot.get(i);
8839                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
8840                                                                TableColumn tableColumn;
8841                                                                if (!initColumn) {
8842                                                                        if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
8843                                                                                if (tableModel.getColumns().size() <= i) {
8844                                                                                        continue;
8845                                                                                }
8846                                                                                tableColumn = tableModel.getColumns().get(i);
8847                                                                        } else {
8848                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
8849                                                                                                (TObjectName) resultColumn.getColumnObject());
8850                                                                        }
8851                                                                        if (containStarColumn(tableColumns)) {
8852                                                                                getStarColumn(tableColumns)
8853                                                                                                .bindStarLinkColumn((TObjectName) resultColumn.getColumnObject());
8854                                                                        }
8855                                                                } else {
8856                                                                        TObjectName matchedColumnName = (TObjectName) resultColumn.getColumnObject();
8857                                                                        tableColumn = matchColumn(tableColumns, matchedColumnName);
8858                                                                        if (tableColumn == null) {
8859                                                                                if (!isEmptyCollection(valueMap)) {
8860                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
8861                                                                                        if (index != -1) {
8862                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
8863                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
8864                                                                                                } else if (isEmptyCollection(keyMap) && index < tableColumns.size()) {
8865                                                                                                        tableColumn = tableColumns.get(index);
8866                                                                                                } else {
8867                                                                                                        continue;
8868                                                                                                }
8869                                                                                        } else {
8870                                                                                                continue;
8871                                                                                        }
8872                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
8873                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
8874                                                                                } else if (isEmptyCollection(keyMap) && isEmptyCollection(valueMap)
8875                                                                                                && i < tableColumns.size()) {
8876                                                                                        tableColumn = tableColumns.get(i);
8877                                                                                } else {
8878                                                                                        continue;
8879                                                                                }
8880                                                                        }
8881                                                                }
8882
8883                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8884                                                                relation.setEffectType(effectType);
8885                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8886                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8887                                                                Process process = modelFactory.createProcess(stmt);
8888                                                                relation.setProcess(process);
8889                                                        } else {
8890                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
8891                                                                if (alias != null && alias.getAliasName() != null) {
8892                                                                        TableColumn tableColumn;
8893                                                                        if (!initColumn) {
8894                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
8895                                                                                        if (tableModel.getColumns().size() <= i) {
8896                                                                                                if (tableModel.isPath()) {
8897                                                                                                        tableColumn = tableModel.getColumns().get(0);
8898                                                                                                } else {
8899                                                                                                        continue;
8900                                                                                                }
8901                                                                                        } else {
8902                                                                                                tableColumn = tableModel.getColumns().get(i);
8903                                                                                        }
8904                                                                                } else {
8905                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
8906                                                                                                        alias.getAliasName());
8907                                                                                        if (containStarColumn(resultSetModel)) {
8908                                                                                                tableColumn.notBindStarLinkColumn(true);
8909                                                                                        }
8910                                                                                }
8911                                                                                if (containStarColumn(tableColumns)) {
8912                                                                                        getStarColumn(tableColumns).bindStarLinkColumn(alias.getAliasName());
8913                                                                                }
8914                                                                        } else {
8915                                                                                TObjectName matchedColumnName = alias.getAliasName();
8916                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
8917                                                                                if (tableColumn == null) {
8918                                                                                        if (!isEmptyCollection(valueMap)) {
8919                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
8920                                                                                                if (index != -1) {
8921                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
8922                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
8923                                                                                                        } else if (isEmptyCollection(keyMap)
8924                                                                                                                        && index < tableColumns.size()) {
8925                                                                                                                tableColumn = tableColumns.get(index);
8926                                                                                                        } else {
8927                                                                                                                continue;
8928                                                                                                        }
8929                                                                                                } else {
8930                                                                                                        continue;
8931                                                                                                }
8932                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
8933                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
8934                                                                                        } else {
8935                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
8936                                                                                                                alias.getAliasName());
8937                                                                                        }
8938                                                                                }
8939
8940                                                                        }
8941
8942                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8943                                                                        relation.setEffectType(effectType);
8944                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8945                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8946                                                                        Process process = modelFactory.createProcess(stmt);
8947                                                                        relation.setProcess(process);
8948
8949                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getFieldAttr() != null) {
8950                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
8951                                                                                        .getFieldAttr();
8952
8953                                                                        Object model = modelManager.getModel( resultColumn.getColumnObject());
8954                                                                        
8955                                                                        TableColumn tableColumn;
8956                                                                        if (!initColumn) {
8957                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
8958                                                                                        if (fieldAttr.toString().endsWith("*")) {
8959                                                                                                int starIndex = 0;
8960                                                                                                for (TableColumn column : tableModel.getColumns()) {
8961                                                                                                        starIndex++;
8962                                                                                                        DataFlowRelationship relation = modelFactory
8963                                                                                                                        .createDataFlowRelation();
8964                                                                                                        relation.setEffectType(effectType);
8965                                                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
8966                                                                                                        if (resultColumn.getStarLinkColumnList().size() == tableModel
8967                                                                                                                        .getColumns().size()) {
8968                                                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
8969                                                                                                                                resultSetModel,
8970                                                                                                                                resultColumn.getStarLinkColumnList().get(starIndex - 1),
8971                                                                                                                                false);
8972                                                                                                                relation.addSource(
8973                                                                                                                                new ResultColumnRelationshipElement(expandStarColumn));
8974                                                                                                        } else {
8975                                                                                                                relation.addSource(
8976                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
8977                                                                                                        }
8978                                                                                                        Process process = modelFactory.createProcess(stmt);
8979                                                                                                        relation.setProcess(process);
8980                                                                                                }
8981                                                                                                continue;
8982                                                                                        }
8983                                                                                        if (tableModel.getColumns().size() <= i) {
8984                                                                                                continue;
8985                                                                                        }
8986                                                                                        tableColumn = tableModel.getColumns().get(i);
8987                                                                                } else {
8988                                                                                        if(model instanceof LinkedHashMap) {
8989                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
8990                                                                                                for(String key: resultColumns.keySet()) {
8991                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
8992                                                                                                        DataFlowRelationship relation = modelFactory
8993                                                                                                                        .createDataFlowRelation();
8994                                                                                                        relation.setEffectType(effectType);
8995                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8996                                                                                                        relation.addSource(
8997                                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
8998                                                                                                        Process process = modelFactory.createProcess(stmt);
8999                                                                                                        relation.setProcess(process);
9000                                                                                                }
9001                                                                                                continue;
9002                                                                                        }
9003                                                                                        else {
9004                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, fieldAttr);
9005                                                                                        }
9006                                                                                }
9007                                                                        } else if (tableModel.isDetermined() && i < tableModel.getColumns().size()) {
9008                                                                                if (!isEmptyCollection(valueMap)) {
9009                                                                                        TObjectName matchedColumnName = fieldAttr;
9010                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
9011                                                                                        if (index != -1) {
9012                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9013                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
9014                                                                                                } else if (isEmptyCollection(keyMap)
9015                                                                                                                && index < tableColumns.size()) {
9016                                                                                                        tableColumn = tableColumns.get(index);
9017                                                                                                } else {
9018                                                                                                        continue;
9019                                                                                                }
9020                                                                                        } else {
9021                                                                                                continue;
9022                                                                                        }
9023                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9024                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
9025                                                                                } else {
9026                                                                                        tableColumn = tableModel.getColumns().get(i);
9027                                                                                }
9028                                                                        } else {
9029                                                                                TObjectName matchedColumnName = fieldAttr;
9030                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
9031                                                                                if (tableColumn == null) {
9032                                                                                        if (!isEmptyCollection(valueMap)) {
9033                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
9034                                                                                                if (index != -1) {
9035                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9036                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
9037                                                                                                        } else if (isEmptyCollection(keyMap)
9038                                                                                                                        && index < tableColumns.size()) {
9039                                                                                                                tableColumn = tableColumns.get(index);
9040                                                                                                        } else {
9041                                                                                                                continue;
9042                                                                                                        }
9043                                                                                                } else {
9044                                                                                                        continue;
9045                                                                                                }
9046                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9047                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
9048                                                                                        } else {
9049                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9050                                                                                                                fieldAttr);
9051                                                                                        }
9052                                                                                }
9053                                                                        }
9054
9055                                                                        if (!"*".equals(getColumnName(tableColumn.getColumnObject()))
9056                                                                                        && "*".equals(getColumnName(fieldAttr))) {
9057                                                                                TObjectName columnObject = fieldAttr;
9058                                                                                TTable sourceTable = columnObject.getSourceTable();
9059                                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
9060                                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
9061                                                                                        for (int j = 0; j < columns.length; j++) {
9062                                                                                                TObjectName columnName = columns[j];
9063                                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
9064                                                                                                        continue;
9065                                                                                                }
9066                                                                                                resultColumn.bindStarLinkColumn(columnName);
9067                                                                                        }
9068                                                                                } else {
9069                                                                                        TTableList tables = stmt.getTables();
9070                                                                                        for (int k = 0; k < tables.size(); k++) {
9071                                                                                                TTable tableElement = tables.getTable(k);
9072                                                                                                TObjectName[] columns = modelManager.getTableColumns(tableElement);
9073                                                                                                for (int j = 0; j < columns.length; j++) {
9074                                                                                                        TObjectName columnName = columns[j];
9075                                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
9076                                                                                                                continue;
9077                                                                                                        }
9078                                                                                                        resultColumn.bindStarLinkColumn(columnName);
9079                                                                                                }
9080                                                                                        }
9081                                                                                }
9082                                                                        }
9083
9084                                                                        if ("*".equals(getColumnName(tableColumn.getColumnObject())) && resultColumn != null
9085                                                                                        && !resultColumn.getStarLinkColumns().isEmpty()) {
9086                                                                                tableColumn.bindStarLinkColumns(resultColumn.getStarLinkColumns());
9087                                                                        }
9088
9089                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9090                                                                        relation.setEffectType(effectType);
9091                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9092                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9093
9094                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
9095                                                                                tableColumn.getTable().setStarStmt("insert");
9096                                                                        }
9097
9098                                                                        Process process = modelFactory.createProcess(stmt);
9099                                                                        relation.setProcess(process);
9100                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9101                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
9102                                                                        if (!initColumn) {
9103                                                                                TableColumn tableColumn;
9104                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9105                                                                                        if (tableModel.getColumns().size() <= i) {
9106                                                                                                continue;
9107                                                                                        }
9108                                                                                        tableColumn = tableModel.getColumns().get(i);
9109                                                                                } else {
9110                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9111                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9112                                                                                                                        .getConstantOperand(),
9113                                                                                                        i);
9114                                                                                }
9115
9116                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
9117                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
9118                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
9119                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
9120                                                                                        if (schema != null) {
9121                                                                                                TSQLTable tempTable = schema.createTable(
9122                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
9123                                                                                                tempTable.addColumn(tableColumn.getName());
9124                                                                                        }
9125                                                                                }
9126                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9127                                                                                relation.setEffectType(effectType);
9128                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9129                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9130                                                                                Process process = modelFactory.createProcess(stmt);
9131                                                                                relation.setProcess(process);
9132                                                                        } else {
9133                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9134                                                                                relation.setEffectType(effectType);
9135                                                                                relation.setTarget(
9136                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
9137                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9138                                                                                Process process = modelFactory.createProcess(stmt);
9139                                                                                relation.setProcess(process);
9140                                                                        }
9141                                                                } else {
9142                                                                        if (!initColumn) {
9143                                                                                TableColumn tableColumn;
9144                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9145                                                                                        if (tableModel.getColumns().size() <= i) {
9146                                                                                                continue;
9147                                                                                        }
9148                                                                                        tableColumn = tableModel.getColumns().get(i);
9149                                                                                } else {
9150                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9151                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
9152                                                                                }
9153                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
9154                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
9155                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
9156                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
9157                                                                                        if (schema != null) {
9158                                                                                                TSQLTable tempTable = schema.createTable(
9159                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
9160                                                                                                tempTable.addColumn(tableColumn.getName());
9161                                                                                        }
9162                                                                                }
9163                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9164                                                                                relation.setEffectType(effectType);
9165                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9166                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9167                                                                                Process process = modelFactory.createProcess(stmt);
9168                                                                                relation.setProcess(process);
9169                                                                        } else {
9170                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9171                                                                                relation.setEffectType(effectType);
9172                                                                                relation.setTarget(
9173                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
9174                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9175                                                                                Process process = modelFactory.createProcess(stmt);
9176                                                                                relation.setProcess(process);
9177                                                                        }
9178                                                                }
9179                                                        }
9180                                                }
9181                                        } else if (stmt.getSubQuery() != null) {
9182                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
9183                                                if (resultSetModel != null) {
9184
9185                                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
9186                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
9187                                                                impactRelation.setEffectType(effectType);
9188                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
9189                                                                                resultSetModel.getRelationRows()));
9190                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
9191                                                                                tableModel.getRelationRows()));
9192                                                        }
9193
9194                                                        if(stmt.getColumnList()!=null && stmt.getColumnList().size()>0) {
9195                                                                for(int i=0;i<stmt.getColumnList().size();i++) {
9196                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9197                                                                                        stmt.getColumnList().getObjectName(i));
9198                                                                }
9199                                                                
9200                                                                List<TableColumn> columns = tableModel.getColumns();
9201                                                                int resultSetSize = resultSetModel.getColumns().size();
9202                                                                int starIndex = 0; 
9203                                                                int j = 0;
9204                                                                boolean fromStruct = false;
9205                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
9206                                                                        String column = columns.get(i).getName();
9207                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
9208                                                                        if (!resultColumn.getName().contains("*")) {
9209                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
9210                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
9211                                                                                                && resultSetSize == 1) {
9212                                                                                        starIndex++;
9213                                                                                        if (resultSetSize - j == columns.size() - i) {
9214                                                                                                j++;
9215
9216                                                                                        }
9217                                                                                }
9218                                                                                else {
9219                                                                                        j++;
9220                                                                                }
9221                                                                        } else {
9222                                                                                starIndex++;
9223                                                                                if (resultSetSize - j == columns.size() - i) {
9224                                                                                        j++;
9225
9226                                                                                }
9227                                                                        }
9228                                                                        if (column != null) {
9229                                                                                TableColumn tableColumn;
9230                                                                                // if (!initColumn) {
9231                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
9232                                                                                if (tableColumn == null) {
9233                                                                                        if (tableModel.isCreateTable()
9234                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9235                                                                                                if (tableModel.getColumns().size() <= i) {
9236                                                                                                        continue;
9237                                                                                                }
9238                                                                                                tableColumn = tableModel.getColumns().get(i);
9239                                                                                        } else {
9240                                                                                                TObjectName columnName = new TObjectName();
9241                                                                                                columnName.setString(column);
9242                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
9243                                                                                                                false);
9244                                                                                        }
9245                                                                                }
9246                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
9247                                                                                        j--;
9248                                                                                        fromStruct = true;
9249                                                                                }
9250                                                                                if(fromStruct && !tableColumn.isStruct()) {
9251                                                                                        fromStruct = false;
9252                                                                                        resultColumn = resultSetModel.getColumns().get(j);
9253                                                                                        j++;
9254                                                                                }
9255                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9256                                                                                relation.setEffectType(effectType);
9257                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9258                                                                                if (resultColumn.hasStarLinkColumn()
9259                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
9260                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
9261                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
9262                                                                                                        false);
9263                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
9264                                                                                } else {
9265                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
9266                                                                                }
9267                                                                                Process process = modelFactory.createProcess(stmt);
9268                                                                                relation.setProcess(process);
9269                                                                        }
9270                                                                }
9271                                                        }
9272                                                        else {
9273                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
9274                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9275                                                                        TAliasClause alias = null;
9276                                                                        if(resultColumn.getColumnObject() instanceof TResultColumn) {
9277                                                                                alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
9278                                                                        }
9279                                                                        if (stmt.getColumnList() != null) {
9280                                                                                if (i < stmt.getColumnList().size()) {
9281                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9282                                                                                        relation.setEffectType(effectType);
9283                                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9284                                                                                                        stmt.getColumnList().getObjectName(i));
9285                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9286                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9287                                                                                        Process process = modelFactory.createProcess(stmt);
9288                                                                                        relation.setProcess(process);
9289                                                                                }
9290                                                                        } else {
9291                                                                                if (alias != null && alias.getAliasName() != null) {
9292                                                                                        TableColumn tableColumn;
9293                                                                                        if (!initColumn) {
9294                                                                                                if (tableModel.isCreateTable()
9295                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9296                                                                                                        if (tableModel.getColumns().size() <= i) {
9297                                                                                                                continue;
9298                                                                                                        }
9299                                                                                                        tableColumn = tableModel.getColumns().get(i);
9300                                                                                                } else {
9301                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9302                                                                                                                        alias.getAliasName());
9303                                                                                                }
9304                                                                                        } else {
9305                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
9306                                                                                                if (tableColumn == null) {
9307                                                                                                        continue;
9308                                                                                                }
9309                                                                                        }
9310        
9311                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9312                                                                                        relation.setEffectType(effectType);
9313                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9314                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9315                                                                                        Process process = modelFactory.createProcess(stmt);
9316                                                                                        relation.setProcess(process);
9317                                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName 
9318                                                                                                || ( resultColumn.getColumnObject() instanceof TResultColumn && ((TResultColumn) resultColumn.getColumnObject())
9319                                                                                                .getFieldAttr() != null)) {
9320                                                                                        TObjectName fieldAttr = null;
9321                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
9322                                                                                                fieldAttr = (TObjectName)resultColumn.getColumnObject();
9323                                                                                        }
9324                                                                                        else if (resultColumn.getColumnObject() instanceof TResultColumn) {
9325                                                                                                fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
9326                                                                                                                .getFieldAttr();
9327                                                                                        }
9328                                
9329                                                                                        TableColumn tableColumn;
9330                                                                                        if (!initColumn) {
9331                                                                                                if (tableModel.isCreateTable()
9332                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9333                                                                                                        if (tableModel.getColumns().size() <= i) {
9334                                                                                                                continue;
9335                                                                                                        }
9336                                                                                                        tableColumn = tableModel.getColumns().get(i);
9337                                                                                                } else {
9338                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9339                                                                                                                        fieldAttr);
9340                                                                                                }
9341                                                                                        } else {
9342                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
9343                                                                                                if (tableColumn == null) {
9344                                                                                                        continue;
9345                                                                                                }
9346                                                                                        }
9347        
9348                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9349                                                                                        relation.setEffectType(effectType);
9350                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9351                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9352                                                                                        Process process = modelFactory.createProcess(stmt);
9353                                                                                        relation.setProcess(process);
9354                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9355                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
9356                                                                                        if (!initColumn) {
9357                                                                                                TableColumn tableColumn;
9358                                                                                                if (tableModel.isCreateTable()
9359                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9360                                                                                                        if (tableModel.getColumns().size() <= i) {
9361                                                                                                                continue;
9362                                                                                                        }
9363                                                                                                        tableColumn = tableModel.getColumns().get(i);
9364                                                                                                } else {
9365                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9366                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9367                                                                                                                                        .getConstantOperand(),
9368                                                                                                                        i);
9369                                                                                                }
9370                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9371                                                                                                relation.setEffectType(effectType);
9372                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9373                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9374                                                                                                Process process = modelFactory.createProcess(stmt);
9375                                                                                                relation.setProcess(process);
9376                                                                                        }
9377                                                                                } else {
9378                                                                                        if (!initColumn) {
9379                                                                                                TableColumn tableColumn;
9380                                                                                                if (tableModel.isCreateTable()
9381                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9382                                                                                                        if (tableModel.getColumns().size() <= i) {
9383                                                                                                                continue;
9384                                                                                                        }
9385                                                                                                        tableColumn = tableModel.getColumns().get(i);
9386                                                                                                } else {
9387                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9388                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
9389                                                                                                }
9390                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9391                                                                                                relation.setEffectType(effectType);
9392                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9393                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9394                                                                                                Process process = modelFactory.createProcess(stmt);
9395                                                                                                relation.setProcess(process);
9396                                                                                        }
9397                                                                                }
9398                                                                        }
9399                                                                }
9400                                                        }
9401                                                }
9402                                        }
9403                                }
9404                        } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
9405                                TObjectNameList items = stmt.getColumnList();
9406                                TMultiTargetList values = stmt.getValues();
9407                                if (values != null) {
9408                                        for (int k = 0; values != null && k < values.size(); k++) {
9409                                                int j = 0;
9410                                                for (int i = 0; i < items.size(); i++) {
9411                                                        TObjectName column = items.getObjectName(i);
9412                                                        TableColumn tableColumn;
9413                                                        if (!initColumn) {
9414                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9415                                                                        if (tableModel.getColumns().size() <= i) {
9416                                                                                continue;
9417                                                                        }
9418                                                                        tableColumn = tableModel.getColumns().get(i);
9419                                                                } else {
9420                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column);
9421                                                                }
9422                                                        } else {
9423                                                                tableColumn = matchColumn(tableColumns, column);
9424                                                                if (tableColumn == null) {
9425                                                                        continue;
9426                                                                }
9427                                                        }
9428                                                        TResultColumn columnObject = values.getMultiTarget(k).getColumnList().getResultColumn(j);
9429                                                        if (columnObject == null) {
9430                                                                continue;
9431                                                        }
9432                                                        TExpression valueExpr = columnObject.getExpr();
9433                                                        columnsInExpr visitor = new columnsInExpr();
9434                                                        valueExpr.inOrderTraverse(visitor);
9435                                                        List<TObjectName> objectNames = visitor.getObjectNames();
9436                                                        List<TParseTreeNode> constants = visitor.getConstants();
9437                                                        List<TParseTreeNode> functions = visitor.getFunctions();
9438                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9439
9440                                                        Process process = modelFactory.createProcess(stmt);
9441
9442
9443
9444                                                        if (functions != null && !functions.isEmpty()) {
9445                                                                analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
9446                                                        }
9447
9448                                                        if (subquerys != null && !subquerys.isEmpty()) {
9449                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
9450                                                        }
9451                                                        if (objectNames != null && !objectNames.isEmpty()) {
9452                                                                analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
9453                                                                                process, i);
9454                                                        }
9455                                                        //insert into values generate too many constant relations, ignore constant relations.
9456                                                        if (constants != null && !constants.isEmpty()) {
9457                                                                if (!option.isIgnoreInsertIntoValues() || stmt.getParentStmt() != null) {
9458                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType,
9459                                                                                        functions, process);
9460                                                                }
9461                                                        }
9462                                                        j++;
9463                                                }
9464                                        }
9465                                } else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
9466                                        analyzeCustomSqlStmt(stmt.getExecuteStmt());
9467                                        Procedure procedure = modelManager.getProcedureByName(DlineageUtil
9468                                                        .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
9469                                        if (procedure!=null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) {
9470                                                TStoredProcedureSqlStatement procedureStmt = (TStoredProcedureSqlStatement) procedure
9471                                                                .getProcedureObject();
9472                                                List<TSelectSqlStatement> stmtItems = getLastSelectStmt(procedureStmt);
9473                                                if (stmtItems != null) {
9474                                                        for(TSelectSqlStatement stmtItem: stmtItems) {
9475                                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmtItem);
9476                                                                if (resultSetModel != null) {
9477                                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
9478                                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9479                                                                                Transform transform = new Transform();
9480                                                                                transform.setType(Transform.FUNCTION);
9481                                                                                transform.setCode(stmt.getExecuteStmt().getModuleName());
9482                                                                                resultColumn.setTransform(transform);
9483                                                                                
9484                                                                                if (stmt.getColumnList() != null) {
9485                                                                                        if (i < stmt.getColumnList().size()) {
9486                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9487                                                                                                relation.setEffectType(effectType);
9488                                                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9489                                                                                                                stmt.getColumnList().getObjectName(i));                                                                                         
9490                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9491                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9492                                                                                                Process process = modelFactory.createProcess(stmt);
9493                                                                                                relation.setProcess(process);                                   
9494                                                                                        }
9495                                                                                } else {
9496                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
9497                                                                                                TObjectName fieldAttr = ((TObjectName) resultColumn.getColumnObject());
9498                                                                                                TableColumn tableColumn;
9499                                                                                                if (!initColumn) {
9500                                                                                                        if (tableModel.isCreateTable()
9501                                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9502                                                                                                                if (tableModel.getColumns().size() <= i) {
9503                                                                                                                        continue;
9504                                                                                                                }
9505                                                                                                                tableColumn = tableModel.getColumns().get(i);
9506                                                                                                        } else {
9507                                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9508                                                                                                                                fieldAttr);
9509                                                                                                        }
9510                                                                                                } else {
9511                                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
9512                                                                                                        if (tableColumn == null) {
9513                                                                                                                continue;
9514                                                                                                        }
9515                                                                                                }
9516        
9517                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9518                                                                                                relation.setEffectType(effectType);
9519                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9520                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9521                                                                                                Process process = modelFactory.createProcess(stmt);
9522                                                                                                relation.setProcess(process);
9523
9524                                                                                                
9525                                                                                        } else {
9526                                                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject())
9527                                                                                                                .getAliasClause();
9528                                                                                                if (alias != null && alias.getAliasName() != null) {
9529                                                                                                        TableColumn tableColumn;
9530                                                                                                        if (!initColumn) {
9531                                                                                                                if (tableModel.isCreateTable()
9532                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9533                                                                                                                        if (tableModel.getColumns().size() <= i) {
9534                                                                                                                                continue;
9535                                                                                                                        }
9536                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9537                                                                                                                } else {
9538                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9539                                                                                                                                        alias.getAliasName());
9540                                                                                                                }
9541                                                                                                        } else {
9542                                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
9543                                                                                                                if (tableColumn == null) {
9544                                                                                                                        continue;
9545                                                                                                                }
9546                                                                                                        }
9547                                                                                                        
9548                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9549                                                                                                        relation.setEffectType(effectType);
9550                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9551                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9552                                                                                                        Process process = modelFactory.createProcess(stmt);
9553                                                                                                        relation.setProcess(process);
9554                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject())
9555                                                                                                                .getFieldAttr() != null) {
9556                                                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
9557                                                                                                                        .getFieldAttr();
9558                                                                                                        TableColumn tableColumn;
9559                                                                                                        if (!initColumn) {
9560                                                                                                                if (tableModel.isCreateTable()
9561                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9562                                                                                                                        if (tableModel.getColumns().size() <= i) {
9563                                                                                                                                continue;
9564                                                                                                                        }
9565                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9566                                                                                                                } else {
9567                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9568                                                                                                                                        fieldAttr);
9569                                                                                                                }
9570                                                                                                        } else {
9571                                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
9572                                                                                                                if (tableColumn == null) {
9573                                                                                                                        continue;
9574                                                                                                                }
9575                                                                                                        }
9576                                                                                                        
9577                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9578                                                                                                        relation.setEffectType(effectType);
9579                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9580                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9581                                                                                                        Process process = modelFactory.createProcess(stmt);
9582                                                                                                        relation.setProcess(process);                                                                                                   
9583                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9584                                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
9585                                                                                                        if (!initColumn) {
9586                                                                                                                TableColumn tableColumn;
9587                                                                                                                if (tableModel.isCreateTable()
9588                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9589                                                                                                                        if (tableModel.getColumns().size() <= i) {
9590                                                                                                                                continue;
9591                                                                                                                        }
9592                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9593                                                                                                                } else {
9594                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9595                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9596                                                                                                                                                        .getConstantOperand(),
9597                                                                                                                                        i);
9598                                                                                                                }
9599                                                                                                                
9600                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9601                                                                                                                relation.setEffectType(effectType);
9602                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9603                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9604                                                                                                                Process process = modelFactory.createProcess(stmt);
9605                                                                                                                relation.setProcess(process);
9606                                                                                                        }
9607                                                                                                } else {
9608                                                                                                        if (!initColumn) {
9609                                                                                                                TableColumn tableColumn;
9610                                                                                                                if (tableModel.isCreateTable()
9611                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9612                                                                                                                        if (tableModel.getColumns().size() <= i) {
9613                                                                                                                                continue;
9614                                                                                                                        }
9615                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9616                                                                                                                } else {
9617                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9618                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
9619                                                                                                                                        i);
9620                                                                                                                }
9621                                                                                                                
9622                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9623                                                                                                                relation.setEffectType(effectType);
9624                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9625                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9626                                                                                                                Process process = modelFactory.createProcess(stmt);
9627                                                                                                                relation.setProcess(process);
9628                                                                                                        }
9629                                                                                                }
9630                                                                                        }
9631                                                                                }
9632                                                                        }
9633                                                                }
9634                                                        }
9635                                                }
9636                                        }
9637                                        else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
9638                                                TObjectName functionName = new TObjectName();
9639                                                functionName.setString(procedure.getName());
9640                                                Function function = (Function)createFunction(functionName);
9641                                                if (stmt.getColumnList() != null) {
9642                                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
9643                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9644                                                                relation.setEffectType(effectType);
9645                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9646                                                                                stmt.getColumnList().getObjectName(i));
9647                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9648                                                                relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
9649                                                                Process process = modelFactory.createProcess(stmt);
9650                                                                relation.setProcess(process);
9651                                                        }
9652                                                }
9653                                        }
9654                                }
9655                        } else if (stmt.getValues() != null && stmt.getValues().size() > 0 && tableModel.isCreateTable() && !tableModel.getColumns().isEmpty()) {
9656                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
9657                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
9658                                        boolean allConstant = true;
9659                                        Process process = modelFactory.createProcess(stmt);
9660                                        for (int x = 0; x < columns.size(); x++) {
9661                                                TableColumn tableColumn = tableModel.getColumns().get(x);
9662                                                TResultColumn columnObject = columns.getResultColumn(x);
9663                                                if (columnObject == null) {
9664                                                        continue;
9665                                                }
9666                                                TExpression valueExpr = columnObject.getExpr();
9667                                                columnsInExpr visitor = new columnsInExpr();
9668                                                valueExpr.inOrderTraverse(visitor);
9669                                                List<TObjectName> objectNames = visitor.getObjectNames();
9670                                                List<TParseTreeNode> constants = visitor.getConstants();
9671                                                List<TParseTreeNode> functions = visitor.getFunctions();
9672                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9673
9674                                                if (functions != null && !functions.isEmpty()) {
9675                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
9676                                                        allConstant = false;
9677                                                }
9678
9679                                                if (subquerys != null && !subquerys.isEmpty()) {
9680                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
9681                                                        allConstant = false;
9682                                                }
9683                                                if (objectNames != null && !objectNames.isEmpty()) {
9684                                                        analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
9685                                                                        process);
9686                                                        allConstant = false;
9687                                                }
9688                                                //insert into values generate too many constant relations, ignore constant relations.
9689                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
9690                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType, functions,
9691                                                                        process);
9692                                                        allConstant = false;
9693                                                }
9694                                        }
9695                                        
9696                                        if(allConstant) {
9697                                                modelManager.unbindProcessModel(stmt);
9698                                                tableModel.removeProcess(process);
9699                                        }
9700                                }
9701                        } else if (stmt.getRecordName() != null) {
9702                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
9703                                String variableString = stmt.getRecordName().toString();
9704                                if (variableString.startsWith(":")) {
9705                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
9706                                }
9707                                if (!SQLUtil.isEmpty(procedureName)) {
9708                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
9709                                }
9710                                
9711                                Table recordTable = modelManager
9712                                                .getTableByName(DlineageUtil.getTableFullName(variableString));
9713                                if (recordTable != null) {
9714                                        for (int i = 0; i < recordTable.getColumns().size(); i++) {
9715                                                TableColumn sourceTableColumn = recordTable.getColumns().get(i);
9716                                                TableColumn targetTableColumn = modelFactory.createTableColumn(tableModel,
9717                                                                sourceTableColumn.getColumnObject(), false);
9718                                                if (targetTableColumn != null) {
9719                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9720                                                        relation.setEffectType(effectType);
9721                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
9722                                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
9723                                                        Process process = modelFactory.createProcess(stmt);
9724                                                        relation.setProcess(process);
9725                                                } else if (sourceTableColumn.getName().endsWith("*") && tableModel.isCreateTable()) {
9726                                                        for (TableColumn column : tableModel.getColumns()) {
9727                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9728                                                                relation.setEffectType(effectType);
9729                                                                relation.setTarget(new TableColumnRelationshipElement(column));
9730                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
9731                                                                Process process = modelFactory.createProcess(stmt);
9732                                                                relation.setProcess(process);
9733                                                        }
9734                                                }
9735                                        }
9736                                }
9737                        } else if (stmt.getInsertSource() == EInsertSource.values_function && stmt.getFunctionCall() != null) {
9738                                Table cursor = modelManager.getTableByName(
9739                                                DlineageUtil.getTableFullName(stmt.getFunctionCall().getFunctionName().toString()));
9740                                if (cursor != null) {
9741                                        TObjectName starColumn = new TObjectName();
9742                                        starColumn.setString("*");
9743                                        TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
9744                                        insertColumn.setShowStar(false);
9745                                        insertColumn.setExpandStar(true);
9746                                        for (int j = 0; j < cursor.getColumns().size(); j++) {
9747                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9748                                                dataflowRelation.setEffectType(effectType);
9749                                                dataflowRelation.addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
9750                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(insertColumn));
9751                                                Process process = modelFactory.createProcess(stmt);
9752                                                dataflowRelation.setProcess(process);
9753                                        }
9754                                }
9755
9756                        } else if (stmt.getInsertSource() == EInsertSource.values && stmt.getValues() != null) {
9757                                TObjectName starColumn = new TObjectName();
9758                                starColumn.setString("*");
9759                                TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
9760                                insertColumn.setShowStar(false);
9761                                insertColumn.setExpandStar(true);
9762                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
9763                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
9764                                        for (int x = 0; x < columns.size(); x++) {
9765                                                TResultColumn columnObject = columns.getResultColumn(x);
9766                                                if (columnObject == null) {
9767                                                        continue;
9768                                                }
9769                                                TExpression valueExpr = columnObject.getExpr();
9770                                                columnsInExpr visitor = new columnsInExpr();
9771                                                valueExpr.inOrderTraverse(visitor);
9772                                                List<TObjectName> objectNames = visitor.getObjectNames();
9773                                                List<TParseTreeNode> constants = visitor.getConstants();
9774                                                List<TParseTreeNode> functions = visitor.getFunctions();
9775                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9776
9777                                                Process process = modelFactory.createProcess(stmt);
9778                                                if (functions != null && !functions.isEmpty()) {
9779                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, effectType, process);
9780                                                }
9781
9782                                                if (subquerys != null && !subquerys.isEmpty()) {
9783                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, effectType, process);
9784                                                }
9785                                                if (objectNames != null && !objectNames.isEmpty()) {
9786                                                        analyzeDataFlowRelation(insertColumn, objectNames, null, effectType, functions,
9787                                                                        process);
9788                                                }
9789                                                //insert into values generate too many constant relations, ignore constant relations.
9790                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
9791                                                        analyzeConstantDataFlowRelation(insertColumn, constants, effectType, functions,
9792                                                                        process);
9793                                                }
9794                                        }
9795                                }
9796                        }else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
9797                                analyzeCustomSqlStmt(stmt.getExecuteStmt());
9798                                Procedure procedure = modelManager.getProcedureByName(DlineageUtil
9799                                                .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
9800                                if (procedure!=null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) {
9801                                        TStoredProcedureSqlStatement procedureStmt = (TStoredProcedureSqlStatement) procedure
9802                                                        .getProcedureObject();
9803                                        List<TSelectSqlStatement> stmtItems = getLastSelectStmt(procedureStmt);
9804                                        if (stmtItems != null) {
9805                                                for(TSelectSqlStatement stmtItem: stmtItems) {
9806                                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmtItem);
9807                                                        if (resultSetModel != null) {
9808                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
9809                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9810                                                                        
9811                                                                        Transform transform = new Transform();
9812                                                                        transform.setType(Transform.FUNCTION);
9813                                                                        transform.setCode(stmt.getExecuteStmt().getModuleName());
9814                                                                        resultColumn.setTransform(transform);
9815                                                                        
9816                                                                        TAliasClause alias = null;
9817
9818                                                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
9819                                                                                alias = ((TResultColumn) resultColumn.getColumnObject())
9820                                                                                                .getAliasClause();
9821                                                                        }
9822                                                                        
9823                                                                        if (alias != null && alias.getAliasName() != null) {
9824                                                                                TableColumn tableColumn;
9825                                                                                if (!initColumn) {
9826                                                                                        if (tableModel.isCreateTable()
9827                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9828                                                                                                if(resultColumn.getName().endsWith("*")) {
9829                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
9830                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9831                                                                                                                relation.setEffectType(effectType);
9832                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
9833                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9834                                                                                                                Process process = modelFactory.createProcess(stmt);
9835                                                                                                                relation.setProcess(process);
9836                                                                                                        }
9837                                                                                                        continue;
9838                                                                                                }
9839                                                                                                else {
9840                                                                                                        if (tableModel.getColumns().size() <= i) {
9841                                                                                                                continue;
9842                                                                                                        }
9843                                                                                                        tableColumn = tableModel.getColumns().get(i);
9844                                                                                                }
9845                                                                                        } else {
9846                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9847                                                                                                                alias.getAliasName());
9848                                                                                        }
9849                                                                                } else {
9850                                                                                        tableColumn = matchColumn(tableColumns, alias.getAliasName());
9851                                                                                        if (tableColumn == null) {
9852                                                                                                continue;
9853                                                                                        }
9854                                                                                }
9855                                        
9856                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9857                                                                                relation.setEffectType(effectType);
9858                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9859                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9860                                                                                Process process = modelFactory.createProcess(stmt);
9861                                                                                relation.setProcess(process);
9862                                                                        } else if (((TResultColumn) resultColumn.getColumnObject())
9863                                                                                        .getFieldAttr() != null) {
9864                                                                                TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
9865                                                                                                .getFieldAttr();
9866                                                                                TableColumn tableColumn;
9867                                                                                if (!initColumn) {
9868                                                                                        if (tableModel.isCreateTable()
9869                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9870                                                                                                if(resultColumn.getName().endsWith("*")) {
9871                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
9872                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9873                                                                                                                relation.setEffectType(effectType);
9874                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
9875                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9876                                                                                                                Process process = modelFactory.createProcess(stmt);
9877                                                                                                                relation.setProcess(process);
9878                                                                                                        }
9879                                                                                                        continue;
9880                                                                                                }
9881                                                                                                else {
9882                                                                                                        if (tableModel.getColumns().size() <= i) {
9883                                                                                                                continue;
9884                                                                                                        }
9885                                                                                                        tableColumn = tableModel.getColumns().get(i);
9886                                                                                                }
9887                                                                                        } else {
9888                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9889                                                                                                                fieldAttr);
9890                                                                                        }
9891                                                                                } else {
9892                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
9893                                                                                        if (tableColumn == null) {
9894                                                                                                continue;
9895                                                                                        }
9896                                                                                }
9897                                                                                
9898                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9899                                                                                relation.setEffectType(effectType);
9900                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9901                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9902                                                                                Process process = modelFactory.createProcess(stmt);
9903                                                                                relation.setProcess(process);
9904                                                                        } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9905                                                                                        .getExpressionType() == EExpressionType.simple_constant_t) {
9906                                                                                if (!initColumn) {
9907                                                                                        TableColumn tableColumn;
9908                                                                                        if (tableModel.isCreateTable()
9909                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9910                                                                                                if(resultColumn.getName().endsWith("*")) {
9911                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
9912                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9913                                                                                                                relation.setEffectType(effectType);
9914                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
9915                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9916                                                                                                                Process process = modelFactory.createProcess(stmt);
9917                                                                                                                relation.setProcess(process);
9918                                                                                                        }
9919                                                                                                        continue;
9920                                                                                                }
9921                                                                                                else {
9922                                                                                                        if (tableModel.getColumns().size() <= i) {
9923                                                                                                                continue;
9924                                                                                                        }
9925                                                                                                        tableColumn = tableModel.getColumns().get(i);
9926                                                                                                }
9927                                                                                        } else {
9928                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9929                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9930                                                                                                                                .getConstantOperand(),
9931                                                                                                                i);
9932                                                                                        }
9933                                                                                        
9934                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9935                                                                                        relation.setEffectType(effectType);
9936                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9937                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9938                                                                                        Process process = modelFactory.createProcess(stmt);
9939                                                                                        relation.setProcess(process);
9940                                                                                }
9941                                                                        } else {
9942                                                                                if (!initColumn) {
9943                                                                                        TableColumn tableColumn;
9944                                                                                        if (tableModel.isCreateTable()
9945                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9946                                                                                                if(resultColumn.getName().endsWith("*")) {
9947                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
9948                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9949                                                                                                                relation.setEffectType(effectType);
9950                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
9951                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9952                                                                                                                Process process = modelFactory.createProcess(stmt);
9953                                                                                                                relation.setProcess(process);
9954                                                                                                        }
9955                                                                                                        continue;
9956                                                                                                }
9957                                                                                                else {
9958                                                                                                        if (tableModel.getColumns().size() <= i) {
9959                                                                                                                continue;
9960                                                                                                        }
9961                                                                                                        tableColumn = tableModel.getColumns().get(i);
9962                                                                                                }
9963                                                                                        } else {
9964                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9965                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
9966                                                                                                                i);
9967                                                                                        }
9968                                                                                        
9969                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9970                                                                                        relation.setEffectType(effectType);
9971                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9972                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9973                                                                                        Process process = modelFactory.createProcess(stmt);
9974                                                                                        relation.setProcess(process);
9975                                                                                }
9976                                                                        }
9977                                                                }
9978                                                        }
9979                                                }
9980                                        }
9981                                }
9982                                else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
9983                                        TObjectName functionName = new TObjectName();
9984                                        functionName.setString(procedure.getName());
9985                                        Function function = (Function)createFunction(functionName);
9986                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9987                                        relation.setEffectType(effectType);
9988                                        TObjectName starColumn = new TObjectName();
9989                                        starColumn.setString("*");
9990                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9991                                                        starColumn);
9992                                        tableColumn.setExpandStar(false);
9993                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9994                                        relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
9995                                        Process process = modelFactory.createProcess(stmt);
9996                                        relation.setProcess(process);
9997                                }
9998                        }
9999                }
10000                
10001                if(stmt.getOnDuplicateKeyUpdate()!=null) {
10002                        TTable table = stmt.getTargetTable();
10003                        Table tableModel = modelFactory.createTable(table);
10004                        for(TResultColumn column: stmt.getOnDuplicateKeyUpdate()) {
10005                                if(column.getExpr()==null || column.getExpr().getExpressionType() != EExpressionType.assignment_t) {
10006                                        continue;
10007                                }
10008                                TExpression left = column.getExpr().getLeftOperand();
10009                                TExpression right = column.getExpr().getRightOperand();
10010                                TObjectName columnObject = left.getObjectOperand();
10011                                if (columnObject != null) {
10012                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false);
10013                                        if (tableColumn != null) {
10014                                                columnsInExpr visitor = new columnsInExpr();
10015                                                right.inOrderTraverse(visitor);
10016                                                List<TObjectName> objectNames = visitor.getObjectNames();
10017                                                List<TParseTreeNode> functions = visitor.getFunctions();
10018                                                List<TParseTreeNode> constants = visitor.getConstants();
10019                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10020
10021                                                if (functions != null && !functions.isEmpty()) {
10022                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.update);
10023                                                }
10024                                                if (subquerys != null && !subquerys.isEmpty()) {
10025                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.update);
10026                                                }
10027                                                if (objectNames != null && !objectNames.isEmpty()) {
10028                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.update, functions);
10029                                                }
10030                                                if (constants != null && !constants.isEmpty()) {
10031                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.update, functions);
10032                                                }
10033                                        }
10034                                }
10035                        }
10036                }
10037
10038                if (!expressions.isEmpty() && stmt.getSubQuery() != null) {
10039                        analyzeInsertImpactRelation(stmt.getSubQuery(), tableColumnMap, expressions, effectType);
10040                }
10041        }
10042
10043        private List<TSelectSqlStatement> getLastSelectStmt(TStoredProcedureSqlStatement procedureStmt) {
10044                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
10045                if (procedureStmt.getBodyStatements().size() > 0) {
10046                        for (int j = procedureStmt.getBodyStatements().size() - 1; j >= 0; j--) {
10047                                TCustomSqlStatement stmtItem = procedureStmt.getBodyStatements().get(j);
10048                                if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
10049                                        if (stmtItem.getStatements() != null) {
10050                                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10051                                                if (item != null && !item.isEmpty()) {
10052                                                        stmts.addAll(item);
10053                                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10054                                                                break;
10055                                                        }
10056                                                }
10057                                        }
10058                                        break;
10059                                }
10060                                if (stmtItem instanceof TSelectSqlStatement) {
10061                                        stmts.add((TSelectSqlStatement) stmtItem);
10062                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10063                                                break;
10064                                        }
10065                                } else if (stmtItem.getStatements() != null) {
10066                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10067                                        if (item != null && !item.isEmpty()) {
10068                                                stmts.addAll(item);
10069                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10070                                                        break;
10071                                                }
10072                                        }
10073                                }
10074                        }
10075                }
10076                return stmts;
10077        }
10078
10079        private List<TSelectSqlStatement> getLastSelectStmt(TCustomSqlStatement stmt) {
10080                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
10081                for (int j = stmt.getStatements().size() - 1; j >= 0; j--) {
10082                        TCustomSqlStatement stmtItem = stmt.getStatements().get(j);
10083                        if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
10084                                if (stmtItem.getStatements() != null) {
10085                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10086                                        if (item != null && !item.isEmpty()) {
10087                                                stmts.addAll(item);
10088                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10089                                                        break;
10090                                                }
10091                                        }
10092                                }
10093                                break;
10094                        }
10095                        if (stmtItem instanceof TSelectSqlStatement) {
10096                                stmts.add((TSelectSqlStatement) stmtItem);
10097                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10098                                        break;
10099                                }
10100                        } else if (stmtItem.getStatements() != null) {
10101                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10102                                if (item != null && !item.isEmpty()) {
10103                                        stmts.addAll(item);
10104                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10105                                                break;
10106                                        }
10107                                }
10108                        }
10109                }
10110                return stmts;
10111        }
10112
10113        private TableColumn getStarColumn(List<TableColumn> columns) {
10114                for (TableColumn column : columns) {
10115                        if (column.getName().endsWith("*")) {
10116                                return column;
10117                        }
10118                }
10119                return null;
10120        }
10121
10122        private boolean containStarColumn(List<TableColumn> columns) {
10123                if (columns == null)
10124                        return false;
10125                for (TableColumn column : columns) {
10126                        if (column.getName().endsWith("*")) {
10127                                return true;
10128                        }
10129                }
10130                return false;
10131        }
10132
10133        private boolean containStarColumn(ResultSet resultSet) {
10134                if (resultSet == null || resultSet.getColumns() == null)
10135                        return false;
10136                for (ResultColumn column : resultSet.getColumns()) {
10137                        if (column.getName().endsWith("*")) {
10138                                return true;
10139                        }
10140                }
10141                return false;
10142        }
10143
10144        private int indexOfColumn(List<TResultColumn> columns, TObjectName objectName) {
10145                for (int i = 0; i < columns.size(); i++) {
10146                        if (columns.get(i).toString().trim().equalsIgnoreCase(objectName.toString().trim())) {
10147                                return i;
10148                        }
10149                }
10150                return -1;
10151        }
10152
10153        private boolean isEmptyCollection(Collection<?> keyMap) {
10154                return keyMap == null || keyMap.isEmpty();
10155        }
10156
10157        private void analyzeInsertImpactRelation(TSelectSqlStatement stmt, Map<String, List<TableColumn>> insertMap,
10158                        List<TExpression> expressions, EffectType effectType) {
10159                List<TObjectName> objectNames = new ArrayList<TObjectName>();
10160                for (int i = 0; i < expressions.size(); i++) {
10161                        TExpression condition = expressions.get(i);
10162                        columnsInExpr visitor = new columnsInExpr();
10163                        condition.inOrderTraverse(visitor);
10164                        objectNames.addAll(visitor.getObjectNames());
10165                }
10166
10167                Iterator<String> iter = insertMap.keySet().iterator();
10168                while (iter.hasNext()) {
10169                        String table = iter.next();
10170                        List<TableColumn> tableColumns = insertMap.get(table);
10171                        for (int i = 0; i < tableColumns.size(); i++) {
10172
10173                                TableColumn column = tableColumns.get(i);
10174                                ImpactRelationship relation = modelFactory.createImpactRelation();
10175                                relation.setEffectType(effectType);
10176                                relation.setTarget(new TableColumnRelationshipElement(column));
10177
10178                                for (int j = 0; j < objectNames.size(); j++) {
10179                                        TObjectName columnName = objectNames.get(j);
10180                                        Object model = modelManager.getModel(stmt);
10181                                        if (model instanceof SelectResultSet) {
10182                                                SelectResultSet queryTable = (SelectResultSet) model;
10183                                                List<ResultColumn> columns = queryTable.getColumns();
10184                                                for (int k = 0; k < columns.size(); k++) {
10185                                                        ResultColumn resultColumn = columns.get(k);
10186                                                        if (resultColumn.getAlias() != null
10187                                                                        && columnName.toString().equalsIgnoreCase(resultColumn.getAlias())) {
10188                                                                relation.addSource(
10189                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
10190                                                        } else if (resultColumn.getName() != null
10191                                                                        && columnName.toString().equalsIgnoreCase(resultColumn.getName())) {
10192                                                                relation.addSource(
10193                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
10194                                                        }
10195                                                }
10196                                        }
10197                                }
10198                        }
10199                }
10200        }
10201
10202        private void analyzeUpdateStmt(TUpdateSqlStatement stmt) {
10203                if (stmt.getResultColumnList() == null)
10204                        return;
10205
10206                TTable table = stmt.getTargetTable();
10207                while (table.getCTE() != null || table.getSubquery() != null || (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null)) {
10208                        if (table.getCTE() != null) {
10209                                table = table.getCTE().getSubquery().getTables().getTable(0);
10210                        } else if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) {
10211                                table = table.getLinkTable().getSubquery().getTables().getTable(0);
10212                        } else if (table.getSubquery() != null) {
10213                                table = table.getSubquery().getTables().getTable(0);
10214                        }
10215                }
10216                Table tableModel = modelFactory.createTable(table);
10217                Process process = modelFactory.createProcess(stmt);
10218                tableModel.addProcess(process);
10219
10220                for (int i = 0; i < stmt.tables.size(); i++) {
10221                        TTable tableElement = stmt.tables.getTable(i);
10222                        if (tableElement.getSubquery() != null) {
10223                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
10224                                TSelectSqlStatement subquery = tableElement.getSubquery();
10225                                analyzeSelectStmt(subquery);
10226
10227                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
10228                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
10229                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
10230                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
10231                                                ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
10232                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
10233                                                selectSetRalation.setEffectType(EffectType.select);
10234                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
10235                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
10236                                                selectSetRalation.setProcess(process);
10237                                        }
10238                                }
10239
10240                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(tableElement.getSubquery());
10241                                if (resultSetModel != null && resultSetModel != queryTable
10242                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10243                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10244                                        impactRelation.setEffectType(EffectType.update);
10245                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10246                                                        resultSetModel.getRelationRows()));
10247                                        impactRelation.setTarget(
10248                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(queryTable.getRelationRows()));
10249                                }
10250
10251                        } else if (tableElement.getCTE() != null) {
10252                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
10253
10254                                TObjectNameList cteColumns = tableElement.getCTE().getColumnList();
10255                                if (cteColumns != null) {
10256                                        for (int j = 0; j < cteColumns.size(); j++) {
10257                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
10258                                        }
10259                                }
10260                                TSelectSqlStatement subquery = tableElement.getCTE().getSubquery();
10261                                if (subquery != null && !stmtStack.contains(subquery)) {
10262                                        analyzeSelectStmt(subquery);
10263
10264                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
10265                                        if (resultSetModel != null && resultSetModel != queryTable
10266                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10267                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10268                                                impactRelation.setEffectType(EffectType.select);
10269                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10270                                                                resultSetModel.getRelationRows()));
10271                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10272                                                                queryTable.getRelationRows()));
10273                                        }
10274
10275                                        if (subquery.getSetOperatorType() != ESetOperatorType.none) {
10276                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
10277                                                                .getModel(subquery);
10278                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
10279                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
10280                                                        ResultColumn targetColumn = null;
10281                                                        if (cteColumns != null) {
10282                                                                targetColumn = queryTable.getColumns().get(j);
10283                                                        } else {
10284                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
10285                                                        }
10286                                                        for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) {
10287                                                                for (TObjectName starLinkColumn : starLinkColumns) {
10288                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
10289                                                                }
10290                                                        }
10291                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
10292                                                        selectSetRalation.setEffectType(EffectType.select);
10293                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
10294                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
10295                                                        selectSetRalation.setProcess(process);
10296                                                }
10297                                        } else {
10298                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
10299                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
10300                                                        ResultColumn targetColumn = null;
10301                                                        if (cteColumns != null) {
10302                                                                targetColumn = queryTable.getColumns().get(j);
10303                                                        } else {
10304                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
10305                                                        }
10306                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
10307                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
10308                                                        }
10309                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
10310                                                        selectSetRalation.setEffectType(EffectType.select);
10311                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
10312                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
10313                                                        selectSetRalation.setProcess(process);
10314                                                }
10315                                        }
10316                                } else if (tableElement.getCTE().getUpdateStmt() != null) {
10317                                        analyzeCustomSqlStmt(tableElement.getCTE().getUpdateStmt());
10318                                } else if (tableElement.getCTE().getInsertStmt() != null) {
10319                                        analyzeCustomSqlStmt(tableElement.getCTE().getInsertStmt());
10320                                } else if (tableElement.getCTE().getDeleteStmt() != null) {
10321                                        analyzeCustomSqlStmt(tableElement.getCTE().getDeleteStmt());
10322                                }
10323                        } else {
10324                                modelFactory.createTable(stmt.tables.getTable(i));
10325                        }
10326                }
10327
10328                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
10329                        TResultColumn field = stmt.getResultColumnList().getResultColumn(i);
10330
10331                        if (field.getExpr().getExpressionType() == EExpressionType.function_t) {
10332                                // Handle SQL Server XML modify() method for data lineage
10333                                TFunctionCall funcCall = field.getExpr().getFunctionCall();
10334                                if (funcCall != null && funcCall.getFunctionType() == EFunctionType.xmlmodify_t) {
10335                                        analyzeXmlModifyFunction(stmt, tableModel, process, funcCall);
10336                                }
10337                                continue;
10338                        }
10339
10340                        TExpression expression = field.getExpr().getLeftOperand();
10341                        if (expression == null) {
10342                                ErrorInfo errorInfo = new ErrorInfo();
10343                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
10344                                errorInfo.setErrorMessage(
10345                                                "Can't get result column expression. Expression is " + field.getExpr().toString());
10346                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(field.getExpr().getStartToken().lineNo,
10347                                                field.getExpr().getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
10348                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(field.getExpr().getEndToken().lineNo,
10349                                                field.getExpr().getEndToken().columnNo + field.getExpr().getEndToken().getAstext().length(),
10350                                                ModelBindingManager.getGlobalHash()));
10351                                errorInfo.fillInfo(this);
10352                                errorInfos.add(errorInfo);
10353                                continue;
10354                        }
10355                        if (expression.getExpressionType() == EExpressionType.list_t) {
10356                                TExpression setExpression = field.getExpr().getRightOperand();
10357                                if (setExpression != null && setExpression.getSubQuery() != null) {
10358                                        TSelectSqlStatement query = setExpression.getSubQuery();
10359                                        analyzeSelectStmt(query);
10360
10361                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
10362                                                        .getModel(query.getResultColumnList());
10363
10364                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10365                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10366                                                impactRelation.setEffectType(EffectType.update);
10367                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10368                                                                resultSetModel.getRelationRows()));
10369                                                impactRelation.setTarget(
10370                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
10371                                        }
10372
10373                                        TExpressionList columnList = expression.getExprList();
10374                                        for (int j = 0; j < columnList.size(); j++) {
10375                                                TObjectName column = columnList.getExpression(j).getObjectOperand();
10376
10377                                                if (column.getDbObjectType() == EDbObjectType.variable) {
10378                                                        continue;
10379                                                }
10380
10381                                                if (column.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
10382                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
10383                                                        continue;
10384                                                }
10385
10386                                                if (column.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
10387                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
10388                                                        continue;
10389                                                }
10390
10391                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
10392                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column, false);
10393                                                if (tableColumn != null) {
10394                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10395                                                        relation.setEffectType(EffectType.update);
10396                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10397                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10398                                                        relation.setProcess(process);
10399                                                }
10400
10401                                        }
10402                                }
10403                        } else if (expression.getExpressionType() == EExpressionType.simple_object_name_t) {
10404                                TExpression setExpression = field.getExpr().getRightOperand();
10405                                if (setExpression != null && setExpression.getSubQuery() != null) {
10406                                        TSelectSqlStatement query = setExpression.getSubQuery();
10407                                        analyzeSelectStmt(query);
10408
10409                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
10410                                                        .getModel(query.getResultColumnList());
10411
10412                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10413                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10414                                                impactRelation.setEffectType(EffectType.update);
10415                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10416                                                                resultSetModel.getRelationRows()));
10417                                                impactRelation.setTarget(
10418                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
10419                                        }
10420
10421                                        TObjectName column = expression.getObjectOperand();
10422                                        ResultColumn resultColumn = resultSetModel.getColumns().get(0);
10423                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column, false);
10424                                        if (tableColumn != null) {
10425                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10426                                                relation.setEffectType(EffectType.update);
10427                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10428                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10429                                                relation.setProcess(process);
10430                                        }
10431                                } else if (setExpression != null) {
10432                                        // ResultSet resultSet = modelFactory.createResultSet(stmt,
10433                                        // true);
10434
10435                                        ResultSet resultSet = modelFactory.createResultSet(stmt, false);
10436
10437                                        createPseudoImpactRelation(stmt, resultSet, EffectType.update);
10438
10439                                        TObjectName columnObject = expression.getObjectOperand();
10440
10441                                        ResultColumn updateColumn = modelFactory.createUpdateResultColumn(resultSet, columnObject);
10442
10443                                        columnsInExpr visitor = new columnsInExpr();
10444                                        field.getExpr().getRightOperand().inOrderTraverse(visitor);
10445
10446                                        List<TObjectName> objectNames = visitor.getObjectNames();
10447                                        List<TParseTreeNode> functions = visitor.getFunctions();
10448                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10449
10450                                        if (functions != null && !functions.isEmpty()) {
10451                                                analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.update);
10452                                        }
10453
10454                                        if (subquerys != null && !subquerys.isEmpty()) {
10455                                                analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.update);
10456                                        }
10457
10458                                        Transform transform = new Transform();
10459                                        transform.setType(Transform.EXPRESSION);
10460                                        transform.setCode(setExpression);
10461                                        updateColumn.setTransform(transform);
10462                                        analyzeDataFlowRelation(updateColumn, objectNames, EffectType.update, functions);
10463
10464                                        List<TParseTreeNode> constants = visitor.getConstants();
10465                                        analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.update, functions);
10466
10467                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false);
10468                                        if(tableColumn!=null) {
10469                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10470                                                relation.setEffectType(EffectType.update);
10471                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10472                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
10473                                                relation.setProcess(process);
10474                                        }
10475                                        
10476                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10477                                        impactRelation.setEffectType(EffectType.update);
10478                                        impactRelation.addSource(
10479                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
10480                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
10481                                                        tableModel.getRelationRows()));
10482                                }
10483                        }
10484                }
10485
10486                if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
10487                        for (int i = 0; i < stmt.getJoins().size(); i++) {
10488                                TJoin join = stmt.getJoins().getJoin(i);
10489                                if (join.getJoinItems() != null) {
10490                                        for (int j = 0; j < join.getJoinItems().size(); j++) {
10491                                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
10492                                                TExpression expr = joinItem.getOnCondition();
10493                                                analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on,
10494                                                                EffectType.update);
10495                                        }
10496                                }
10497                        }
10498                }
10499
10500                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
10501                        analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
10502                                        EffectType.update);
10503                }
10504
10505                if (stmt.getOutputClause() != null) {
10506                        TOutputClause outputClause = stmt.getOutputClause();
10507                        if (outputClause.getSelectItemList() != null) {
10508                                ResultSet resultSet = modelFactory.createResultSet(outputClause, false);
10509                                for (int j = 0; j < outputClause.getSelectItemList().size(); j++) {
10510                                        TResultColumn sourceColumn = outputClause.getSelectItemList().getResultColumn(j);
10511                                        ResultColumn sourceColumnModel = modelFactory.createResultColumn(resultSet, sourceColumn);
10512                                        analyzeResultColumn(sourceColumn, EffectType.select);
10513
10514                                        if (outputClause.getIntoTable() != null) {
10515                                                Table intoTableModel = modelFactory.createTableByName(outputClause.getIntoTable());
10516                                                intoTableModel.addProcess(process);
10517                                                if (outputClause.getIntoColumnList() != null) {
10518                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
10519                                                                        outputClause.getIntoColumnList().getObjectName(j));
10520
10521                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10522                                                        relation.setEffectType(EffectType.insert);
10523                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
10524                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
10525                                                } else if (sourceColumn.getAliasClause() != null
10526                                                                || sourceColumn.getExpr().getObjectOperand() != null) {
10527                                                        TObjectName tableColumnObject = null;
10528                                                        if (sourceColumn.getAliasClause() != null) {
10529                                                                tableColumnObject = sourceColumn.getAliasClause().getAliasName();
10530                                                        } else {
10531                                                                tableColumnObject = sourceColumn.getExpr().getObjectOperand();
10532                                                        }
10533
10534                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
10535                                                                        tableColumnObject);
10536                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10537                                                        relation.setEffectType(EffectType.insert);
10538                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
10539                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
10540                                                }
10541                                        }
10542                                }
10543                        }
10544                }
10545        }
10546
10547        /**
10548         * Analyzes SQL Server XML modify() function to extract data lineage from sql:column() references
10549         * in XQuery expressions.
10550         *
10551         * Example SQL:
10552         * SET [Demographics].modify('... sql:column("deleted.LineTotal") ...')
10553         *
10554         * This extracts:
10555         * - Target column: Demographics (from the XML column being modified)
10556         * - Source column: deleted.LineTotal (from sql:column() reference in XQuery)
10557         */
10558        private void analyzeXmlModifyFunction(TUpdateSqlStatement stmt, Table tableModel, Process process, TFunctionCall funcCall) {
10559                TObjectName funcName = funcCall.getFunctionName();
10560                if (funcName == null || funcName.getPartToken() == null) {
10561                        return;
10562                }
10563
10564                // Get the XML column being modified (e.g., [Demographics] from "[Demographics].modify")
10565                String xmlColumnName = funcName.getPartToken().astext;
10566
10567                // Get the XQuery argument
10568                if (funcCall.getArgs() == null || funcCall.getArgs().size() == 0) {
10569                        return;
10570                }
10571
10572                String xqueryString = funcCall.getArgs().getExpression(0).toString();
10573
10574                // Extract sql:column() references from the XQuery string
10575                List<String> sqlColumnRefs = extractSqlColumnReferences(xqueryString);
10576                if (sqlColumnRefs.isEmpty()) {
10577                        return;
10578                }
10579
10580                // Extract XPath target from XQuery (e.g., /IndividualSurvey/TotalPurchaseYTD)
10581                String xpathTarget = extractXPathTarget(xqueryString);
10582
10583                // Build full target column name including XML path
10584                String fullTargetColumnName = xmlColumnName;
10585                if (xpathTarget != null && !xpathTarget.isEmpty()) {
10586                        fullTargetColumnName = xmlColumnName + "." + xpathTarget;
10587                }
10588
10589                // Create the target table column for the XML column being modified
10590                TableColumn targetTableColumn = modelFactory.createInsertTableColumn(tableModel, fullTargetColumnName);
10591
10592                // Create source-to-target relationships for each sql:column reference
10593                for (String sqlColRef : sqlColumnRefs) {
10594                        // Parse the column reference (e.g., "deleted.LineTotal" -> table="deleted", column="LineTotal")
10595                        String[] parts = sqlColRef.split("\\.", 2);
10596                        String sourceTableName = parts.length > 1 ? parts[0] : null;
10597                        String sourceColumnName = parts.length > 1 ? parts[1] : parts[0];
10598
10599                        // Find the source table in the statement's tables
10600                        TTable sourceTable = null;
10601                        if (sourceTableName != null && stmt.tables != null) {
10602                                for (int j = 0; j < stmt.tables.size(); j++) {
10603                                        TTable t = stmt.tables.getTable(j);
10604                                        String tableName = t.getTableName().toString();
10605                                        String alias = t.getAliasName();
10606                                        if (tableName.equalsIgnoreCase(sourceTableName) ||
10607                                                (alias != null && alias.equalsIgnoreCase(sourceTableName))) {
10608                                                sourceTable = t;
10609                                                break;
10610                                        }
10611                                }
10612                        }
10613
10614                        if (sourceTable != null && targetTableColumn != null) {
10615                                // Create a table model for the source if needed
10616                                Table sourceTableModel = modelFactory.createTable(sourceTable);
10617                                TableColumn sourceColumn = modelFactory.createInsertTableColumn(sourceTableModel, sourceColumnName);
10618
10619                                if (sourceColumn != null) {
10620                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10621                                        relation.setEffectType(EffectType.update);
10622                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
10623                                        relation.addSource(new TableColumnRelationshipElement(sourceColumn));
10624                                        relation.setProcess(process);
10625                                        relation.setFunction("modify");
10626                                }
10627                        }
10628                }
10629        }
10630
10631        /**
10632         * Extracts sql:column() references from an XQuery string.
10633         * Example: 'sql:column("deleted.LineTotal")' -> ["deleted.LineTotal"]
10634         */
10635        private List<String> extractSqlColumnReferences(String xquery) {
10636                List<String> refs = new ArrayList<>();
10637                if (xquery == null) {
10638                        return refs;
10639                }
10640
10641                int startIdx = 0;
10642                while ((startIdx = xquery.indexOf("sql:column(", startIdx)) >= 0) {
10643                        int parenStart = startIdx + "sql:column(".length();
10644                        int parenEnd = xquery.indexOf(")", parenStart);
10645                        if (parenEnd < 0) {
10646                                break;
10647                        }
10648
10649                        String arg = xquery.substring(parenStart, parenEnd).trim();
10650                        // Remove quotes (single or double)
10651                        if ((arg.startsWith("\"") && arg.endsWith("\"")) ||
10652                                (arg.startsWith("'") && arg.endsWith("'"))) {
10653                                arg = arg.substring(1, arg.length() - 1);
10654                        }
10655
10656                        if (!arg.isEmpty()) {
10657                                refs.add(arg);
10658                        }
10659
10660                        startIdx = parenEnd + 1;
10661                }
10662
10663                return refs;
10664        }
10665
10666        /**
10667         * Extracts the XPath target from an XQuery modify expression.
10668         * Example: 'replace value of (/IndividualSurvey/TotalPurchaseYTD)[1]' -> "IndividualSurvey.TotalPurchaseYTD"
10669         */
10670        private String extractXPathTarget(String xquery) {
10671                if (xquery == null) {
10672                        return null;
10673                }
10674
10675                // Look for patterns like "(/path/to/element)" or "(/path/to/element)[1]"
10676                int replaceIdx = xquery.indexOf("replace value of");
10677                if (replaceIdx < 0) {
10678                        return null;
10679                }
10680
10681                int parenStart = xquery.indexOf("(/", replaceIdx);
10682                if (parenStart < 0) {
10683                        return null;
10684                }
10685
10686                int parenEnd = xquery.indexOf(")", parenStart);
10687                if (parenEnd < 0) {
10688                        return null;
10689                }
10690
10691                String xpath = xquery.substring(parenStart + 1, parenEnd);
10692                // Remove any predicates like [1]
10693                int bracketIdx = xpath.indexOf("[");
10694                if (bracketIdx > 0) {
10695                        xpath = xpath.substring(0, bracketIdx);
10696                }
10697
10698                // Convert XPath to dot notation (e.g., /IndividualSurvey/TotalPurchaseYTD -> IndividualSurvey.TotalPurchaseYTD)
10699                if (xpath.startsWith("/")) {
10700                        xpath = xpath.substring(1);
10701                }
10702                xpath = xpath.replace("/", ".");
10703
10704                return xpath;
10705        }
10706
10707        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
10708                        EffectType effectType, List<TParseTreeNode> functions) {
10709                analyzeConstantDataFlowRelation(modelObject, constants, effectType, functions, null);
10710        }
10711
10712        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
10713                        EffectType effectType, List<TParseTreeNode> functions, Process process) {
10714                if (constants == null || constants.size() == 0)
10715                        return;
10716
10717                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10718                relation.setEffectType(effectType);
10719                relation.setProcess(process);
10720
10721                if (functions != null && !functions.isEmpty()) {
10722                        relation.setFunction(getFunctionName(functions.get(0)));
10723                }
10724
10725                if (modelObject instanceof ResultColumn) {
10726                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
10727
10728                } else if (modelObject instanceof TableColumn) {
10729                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
10730
10731                } else {
10732                        throw new UnsupportedOperationException();
10733                }
10734
10735                if (option.isShowConstantTable()) {
10736                        Table constantTable = null;
10737                        if(modelObject instanceof FunctionResultColumn && ((FunctionResultColumn)modelObject).getFunction() instanceof TFunctionCall) {
10738                                TFunctionCall function = (TFunctionCall)((FunctionResultColumn)modelObject).getFunction();
10739                                if(function.getFunctionType() == EFunctionType.struct_t) {
10740                                        constantTable = modelFactory.createConstantsTable(String.valueOf(function.toString().hashCode()));
10741                                }
10742                        }
10743                        if(constantTable == null) {
10744                                constantTable = modelFactory.createConstantsTable(stmtStack.peek());
10745                        }
10746                        for (int i = 0; i < constants.size(); i++) {
10747                                TParseTreeNode constant = constants.get(i);
10748                                if (constant instanceof TConstant) {
10749                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TConstant) constant);
10750                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
10751                                } else if (constant instanceof TObjectName) {
10752                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TObjectName) constant,
10753                                                        false);
10754                                        if(constantColumn == null) {
10755                                                continue;
10756                                        }
10757                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
10758                                }
10759                        }
10760                }
10761
10762        }
10763
10764        private String getFunctionName(TParseTreeNode parseTreeNode) {
10765                if (parseTreeNode instanceof TFunctionCall) {
10766                        return ((TFunctionCall) parseTreeNode).getFunctionName().toString();
10767                }
10768                if (parseTreeNode instanceof TCaseExpression) {
10769                        return "case-when";
10770                }
10771                return null;
10772        }
10773
10774        private void analyzeCreateViewStmt(TCustomSqlStatement stmt, TSelectSqlStatement subquery,
10775                        TViewAliasClause viewAlias, TObjectName viewName) {
10776
10777                if (subquery != null) {
10778                        TTableList tables = subquery.getTables();
10779                        if (tables != null) {
10780                                for (int i = 0; i < tables.size(); i++) {
10781                                        TTable table = tables.getTable(i);
10782                                        TCustomSqlStatement createView = viewDDLMap
10783                                                        .get(DlineageUtil.getTableFullName(table.getTableName().toString()));
10784                                        if (createView != null) {
10785                                                analyzeCustomSqlStmt(createView);
10786                                        }
10787                                }
10788                        }
10789                        analyzeSelectStmt(subquery);
10790                }
10791
10792        
10793                if (viewAlias != null && viewAlias.getViewAliasItemList() != null) {
10794                        TViewAliasItemList viewItems = viewAlias.getViewAliasItemList();
10795                        Table viewModel = modelFactory.createView(stmt, viewName, true);
10796                        viewModel.setFromDDL(true);
10797                        viewModel.setDetermined(true);
10798                        Process process = modelFactory.createProcess(stmt);
10799                        viewModel.addProcess(process);
10800                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
10801                        if (resultSetModel != null) {
10802                                int resultSetSize = resultSetModel.getColumns().size();
10803                                int viewItemSize = viewItems.size();
10804                                int j = 0;
10805                                int viewColumnSize = viewItemSize;
10806                                if (resultSetModel.isDetermined() && resultSetSize > viewColumnSize) {
10807                                        viewColumnSize = resultSetSize;
10808                                }
10809                                for (int i = 0; i < viewColumnSize && j < resultSetSize; i++) {
10810                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);                                 
10811                                        if (i < viewItemSize) {
10812                                                TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
10813
10814                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
10815                                                        j++;
10816                                                } else {
10817                                                        if (resultSetSize - j == viewItems.size() - i) {
10818                                                                j++;
10819                                                        }
10820                                                }
10821
10822                                                if (alias != null) {
10823                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
10824                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
10825                                                        if (resultColumn != null) {
10826                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10827                                                                relation.setEffectType(EffectType.create_view);
10828                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10829                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10830                                                                relation.setProcess(process);
10831                                                        }
10832                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
10833                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
10834                                                                        (TObjectName) resultColumn.getColumnObject(), i, true);
10835                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
10836                                                        if (resultColumn != null) {
10837                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10838                                                                relation.setEffectType(EffectType.create_view);
10839                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10840                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10841                                                                relation.setProcess(process);
10842                                                        }
10843                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
10844                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
10845                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), i, true);
10846                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
10847                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
10848                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
10849                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
10850                                                        }
10851                                                        if (resultColumn != null) {
10852                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10853                                                                relation.setEffectType(EffectType.create_view);
10854                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10855                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10856                                                                relation.setProcess(process);
10857                                                        }
10858                                                }
10859                                        }
10860                                        else if(resultSetModel.isDetermined()){
10861                                                TObjectName viewColumnName = new TObjectName();
10862                                                viewColumnName.setString(resultColumn.getName());
10863                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, viewModel.getColumns().size(), true);
10864                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10865                                                relation.setEffectType(EffectType.create_view);
10866                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10867                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10868                                                relation.setProcess(process);
10869                                                j++;
10870                                        }
10871                                }
10872                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10873                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10874                                        impactRelation.setEffectType(EffectType.create_view);
10875                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10876                                                        resultSetModel.getRelationRows()));
10877                                        impactRelation.setTarget(
10878                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
10879                                }
10880                        }
10881
10882                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
10883                                        && subquery.getValueClause().getValueRows().size() == viewItems.size()) {
10884                                for (int i = 0; i < viewItems.size(); i++) {
10885                                        TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
10886
10887                                        if (alias != null) {
10888                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
10889                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
10890                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
10891
10892                                                columnsInExpr visitor = new columnsInExpr();
10893                                                expression.inOrderTraverse(visitor);
10894                                                List<TObjectName> objectNames = visitor.getObjectNames();
10895                                                List<TParseTreeNode> functions = visitor.getFunctions();
10896
10897                                                if (functions != null && !functions.isEmpty()) {
10898                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
10899
10900                                                }
10901
10902                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10903                                                if (subquerys != null && !subquerys.isEmpty()) {
10904                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
10905                                                }
10906
10907                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
10908                                                List<TParseTreeNode> constants = visitor.getConstants();
10909                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
10910                                        }
10911                                }
10912                        }
10913
10914                } else {
10915                        if (viewName == null) {
10916                                ErrorInfo errorInfo = new ErrorInfo();
10917                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
10918                                errorInfo.setErrorMessage("Can't get view name. CreateView is " + stmt.toString());
10919                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
10920                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
10921                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
10922                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
10923                                                ModelBindingManager.getGlobalHash()));
10924                                errorInfo.fillInfo(this);
10925                                errorInfos.add(errorInfo);
10926                                return;
10927                        }
10928                        Table viewModel = modelFactory.createView(stmt, viewName);
10929                        Process process = modelFactory.createProcess(stmt);
10930                        viewModel.addProcess(process);
10931                        if (subquery != null && !subquery.isCombinedQuery()) {
10932                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
10933                                                .getModel(subquery.getResultColumnList());
10934
10935                                boolean determined = false;
10936                                if (!containStarColumn(resultSetModel)) {
10937                                        viewModel.setCreateTable(true);
10938                                        determined = true;
10939                                        viewModel.setFromDDL(true);
10940                                }
10941                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10942                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10943                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
10944                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
10945
10946                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
10947                                                if (alias != null && alias.getAliasName() != null) {
10948                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
10949                                                                        determined);
10950                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10951                                                        relation.setEffectType(EffectType.create_view);
10952                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10953                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10954                                                        relation.setProcess(process);
10955                                                        if (determined) {
10956                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
10957                                                        }
10958                                                } else if (columnObject.getFieldAttr() != null) {
10959                                                        TObjectName viewColumnObject = columnObject.getFieldAttr();
10960                                                        TableColumn viewColumn;
10961                                                        Object model = modelManager.getModel(resultColumn.getColumnObject());
10962                                                        if ("*".equals(viewColumnObject.getColumnNameOnly())) {
10963                                                                if (model instanceof LinkedHashMap) {
10964                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
10965                                                                        LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
10966                                                                        if ("*".equals(columnName)) {
10967                                                                                for (String key : resultColumns.keySet()) {
10968                                                                                        ResultColumn column = resultColumns.get(key);
10969                                                                                        viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
10970                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10971                                                                                        relation.setEffectType(EffectType.create_view);
10972                                                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10973                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
10974                                                                                        relation.setProcess(process);
10975                                                                                }
10976                                                                                continue;
10977                                                                        } else if (resultColumns.containsKey(columnName)) {
10978                                                                                ResultColumn column = resultColumns.get(columnName);
10979                                                                                viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
10980                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10981                                                                                relation.setEffectType(EffectType.create_view);
10982                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10983                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
10984                                                                                relation.setProcess(process);
10985                                                                                continue;
10986                                                                        }
10987                                                                }
10988                                                        }
10989                                                        
10990                                                        if (!SQLUtil.isEmpty(viewColumnObject.getColumnNameOnly())
10991                                                                        && viewColumnObject.getPropertyToken() != null) {
10992                                                                TObjectName object = new TObjectName();
10993                                                                // object.setString(viewColumnObject.getPropertyToken().astext);
10994                                                                object.setPartToken(viewColumnObject.getPropertyToken());
10995                                                                object.setStartToken(viewColumnObject.getPropertyToken());
10996                                                                object.setEndToken(viewColumnObject.getPropertyToken());
10997                                                                viewColumn = modelFactory.createViewColumn(viewModel, object, i, determined);
10998                                                        } else {
10999                                                                viewColumn = modelFactory.createViewColumn(viewModel, columnObject.getFieldAttr(),
11000                                                                                i, determined);
11001                                                        }
11002                                                        
11003                                                        if(determined) {
11004                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11005                                                        }
11006                                                        
11007                                                        if (model instanceof ResultColumn) {
11008                                                                ResultColumn column = (ResultColumn) modelManager
11009                                                                                .getModel(resultColumn.getColumnObject());
11010                                                                if (column != null && !column.getStarLinkColumns().isEmpty()) {
11011                                                                        viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11012                                                                }
11013                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11014                                                                relation.setEffectType(EffectType.create_view);
11015                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11016                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11017                                                                if (sqlenv == null) {
11018                                                                        if (resultColumn.isShowStar()) {
11019                                                                                relation.setShowStarRelation(true);
11020                                                                                viewColumn.setShowStar(true);
11021                                                                                resultColumn.setShowStar(true);
11022                                                                                setSourceShowStar(resultColumn);
11023                                                                        }
11024                                                                }
11025                                                                if (viewColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
11026                                                                        viewModel.setStarStmt("create_view");
11027                                                                }
11028                                                                relation.setProcess(process);
11029                                                        }
11030                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
11031                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
11032                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11033                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
11034                                                        if(determined) {
11035                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11036                                                        }
11037                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11038                                                        relation.setEffectType(EffectType.create_view);
11039                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11040                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11041                                                        relation.setProcess(process);
11042                                                } else {
11043                                                        TGSqlParser parser = columnObject.getGsqlparser();
11044                                                        TObjectName viewColumnName = parser
11045                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
11046                                                        if (viewColumnName == null) {
11047                                                                ErrorInfo errorInfo = new ErrorInfo();
11048                                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
11049                                                                errorInfo.setErrorMessage(
11050                                                                                "Can't parse view column. Column is " + columnObject.toString());
11051                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(
11052                                                                                columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
11053                                                                                ModelBindingManager.getGlobalHash()));
11054                                                                errorInfo
11055                                                                                .setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
11056                                                                                                columnObject.getEndToken().columnNo
11057                                                                                                                + columnObject.getEndToken().getAstext().length(),
11058                                                                                                ModelBindingManager.getGlobalHash()));
11059                                                                errorInfo.fillInfo(this);
11060                                                                errorInfos.add(errorInfo);
11061                                                        } else {
11062                                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
11063                                                                                determined);
11064                                                                if(determined) {
11065                                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11066                                                                }
11067                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11068                                                                relation.setEffectType(EffectType.create_view);
11069                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11070                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11071                                                                relation.setProcess(process);
11072                                                        }
11073                                                }
11074                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11075                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11076                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
11077                                                if(determined) {
11078                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11079                                                }
11080                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11081                                                relation.setEffectType(EffectType.create_view);
11082                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11083                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11084                                                relation.setProcess(process);
11085                                        }
11086                                }
11087                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11088                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11089                                        impactRelation.setEffectType(EffectType.create_view);
11090                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11091                                                        resultSetModel.getRelationRows()));
11092                                        impactRelation.setTarget(
11093                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
11094                                }
11095                        } else if (subquery != null && subquery.isCombinedQuery()) {
11096                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
11097
11098                                boolean determined = false;
11099                                if (!containStarColumn(resultSetModel)) {
11100                                        viewModel.setCreateTable(true);
11101                                        viewModel.setFromDDL(true);
11102                                        determined = true;
11103                                }
11104
11105                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11106                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11107
11108                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
11109                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
11110
11111                                                TAliasClause alias = columnObject.getAliasClause();
11112                                                if (alias != null && alias.getAliasName() != null) {
11113                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
11114                                                                        determined);
11115                                                        if(determined) {
11116                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11117                                                        }
11118                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11119                                                        relation.setEffectType(EffectType.create_view);
11120                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11121                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11122                                                        relation.setProcess(process);
11123                                                } else if (columnObject.getFieldAttr() != null) {
11124                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11125                                                                        columnObject.getFieldAttr(), i, determined);
11126                                                        if(determined) {
11127                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11128                                                        }
11129                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
11130                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
11131                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11132                                                        }
11133                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11134                                                        relation.setEffectType(EffectType.create_view);
11135                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11136                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11137                                                        relation.setProcess(process);
11138                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
11139                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
11140                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11141                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
11142                                                        if(determined) {
11143                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11144                                                        }
11145                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11146                                                        relation.setEffectType(EffectType.create_view);
11147                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11148                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11149                                                        relation.setProcess(process);
11150                                                } else {
11151                                                        TGSqlParser parser = columnObject.getGsqlparser();
11152                                                        TObjectName viewColumnName = parser
11153                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
11154                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
11155                                                                        determined);
11156                                                        if(determined) {
11157                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11158                                                        }
11159                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11160                                                        relation.setEffectType(EffectType.create_view);
11161                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11162                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11163                                                        relation.setProcess(process);
11164                                                }
11165                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11166                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11167                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
11168                                                if(viewColumn!=null) {
11169                                                        if(determined) {
11170                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11171                                                        }
11172                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11173                                                        relation.setEffectType(EffectType.create_view);
11174                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11175                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11176                                                        relation.setProcess(process);
11177                                                }
11178                                        }
11179                                }
11180                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11181                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11182                                        impactRelation.setEffectType(EffectType.create_view);
11183                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11184                                                        resultSetModel.getRelationRows()));
11185                                        impactRelation.setTarget(
11186                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
11187                                }
11188                        }
11189                }
11190        }
11191
11192        private void setSourceShowStar(Object resultColumn) {
11193                for (Relationship relation : modelManager.getRelations()) {
11194                        Collection<RelationshipElement<?>> sources = (Collection<RelationshipElement<?>>)relation.getSources();
11195                        if (relation.getTarget().getElement() == resultColumn && sources != null) {
11196
11197                                ((AbstractRelationship) relation).setShowStarRelation(true);
11198                                for (RelationshipElement<?> source : sources) {
11199                                        Object column = source.getElement();
11200                                        if (column instanceof TableColumn) {
11201                                                if (((TableColumn) column).isShowStar())
11202                                                        continue;
11203                                                ((TableColumn) column).setShowStar(true);
11204                                        }
11205                                        if (column instanceof ResultColumn) {
11206                                                if (((ResultColumn) column).isShowStar())
11207                                                        continue;
11208                                                ((ResultColumn) column).setShowStar(true);
11209                                        }
11210                                        setSourceShowStar(column);
11211                                }
11212                        }
11213                }
11214        }
11215
11216        private String generateQuotedName(TGSqlParser parser, String name) {
11217                return "\"" + name + "\"";
11218        }
11219
11220        private void appendRelations(dataflow dataflow) {
11221                Relationship[] relations = modelManager.getRelations();
11222
11223                appendRelation(dataflow, relations, DataFlowRelationship.class);
11224                appendRelation(dataflow, relations, IndirectImpactRelationship.class);
11225                appendRecordSetRelation(dataflow, relations);
11226                appendCallRelation(dataflow, relations);
11227                appendRelation(dataflow, relations, ImpactRelationship.class);
11228                appendRelation(dataflow, relations, JoinRelationship.class);
11229                appendRelation(dataflow, relations, ERRelationship.class);
11230                appendRelation(dataflow, relations, CrudRelationship.class);
11231        }
11232
11233        private Set<String> appendStarColumns = new HashSet<String>();
11234        private void appendRelation(dataflow dataflow, Relationship[] relations, Class<? extends Relationship> clazz) {
11235                for (int i = 0; i < relations.length; i++) {
11236                        AbstractRelationship relation = (AbstractRelationship) relations[i];
11237                        if (relation.getClass() == clazz) {
11238                                if (relation.getSources() == null || relation.getTarget() == null) {
11239                                        continue;
11240                                }
11241                                Object targetElement = relation.getTarget().getElement();
11242                                TObjectName targetColumnName = null;
11243                                if(relation.getTarget() instanceof ResultColumnRelationshipElement) {
11244                                        targetColumnName = ((ResultColumnRelationshipElement) relation.getTarget()).getColumnName();
11245                                }
11246                                if (targetElement instanceof ResultColumn) {
11247                                        ResultColumn targetColumn = (ResultColumn) targetElement;
11248                                        if (!targetColumn.isPseduo()) 
11249                                        {
11250                                                if (targetColumnName == null) 
11251                                                {
11252                                                        if ("*".equals(targetColumn.getName())) {
11253                                                                updateResultColumnStarLinks(dataflow, relation, -1);
11254                                                        }
11255
11256                                                        if (targetColumn.hasStarLinkColumn()) {
11257                                                                for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
11258                                                                        appendStarRelation(dataflow, relation, j);
11259                                                                }
11260
11261                                                                if (!containsStar(relation.getSources())) {
11262                                                                        continue;
11263                                                                }
11264                                                        }
11265                                                }
11266                                                else {
11267                                                        String columnName = DlineageUtil.getColumnName(targetColumnName);
11268                                                        int index = targetColumn.getStarLinkColumnNames().indexOf(columnName);
11269                                                        if (index != -1) {
11270                                                                int size = targetColumn.getStarLinkColumnNames().size();
11271                                                                if ("*".equals(targetColumn.getName())) {
11272                                                                        if (appendStarColumns.contains(columnName)) {
11273                                                                                updateResultColumnStarLinks(dataflow, relation, index);
11274                                                                        }
11275                                                                        else {
11276                                                                                updateResultColumnStarLinks(dataflow, relation, -1);
11277                                                                                appendStarColumns.add(columnName);
11278                                                                        }
11279                                                                }
11280                                                                appendStarRelation(dataflow, relation, index);
11281                                                                for(int j= size; j < targetColumn.getStarLinkColumnNames().size(); j++) {
11282                                                                        appendStarRelation(dataflow, relation, j);
11283                                                                }
11284                                                                if (!containsStar(relation.getSources())) {
11285                                                                        continue;
11286                                                                }
11287                                                        }
11288                                                }
11289                                        }
11290                                } else if (targetElement instanceof TableColumn) {
11291                                        TableColumn targetColumn = (TableColumn) targetElement;
11292                                        if (!targetColumn.isPseduo()) {
11293                                                if ("*".equals(targetColumn.getName())) {
11294                                                        updateTableColumnStarLinks(dataflow, relation);
11295                                                }
11296
11297                                                if (targetColumn.hasStarLinkColumn() && !targetColumn.isVariant()
11298                                                                && targetColumn.isExpandStar()) {
11299                                                        for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
11300                                                                appendStarRelation(dataflow, relation, j);
11301                                                        }
11302                                                        if (!containsStar(relation.getSources())) {
11303                                                                continue;
11304                                                        }
11305                                                }
11306                                        }
11307                                }
11308
11309                                relationship relationElement = new relationship();
11310                                relationElement.setType(relation.getRelationshipType().name());
11311                                if (relation.getEffectType() != null) {
11312                                        relationElement.setEffectType(relation.getEffectType().name());
11313                                }
11314                                if (relation.getFunction() != null) {
11315                                        relationElement.setFunction(relation.getFunction());
11316                                }
11317                                relationElement.setSqlHash(relation.getSqlHash());
11318                                relationElement.setSqlComment(relation.getSqlComment());
11319
11320                                if (relation.getProcedureId() != null) {
11321                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
11322                                }
11323                                relationElement.setId(String.valueOf(relation.getId()));
11324                                if (relation.getProcess() != null) {
11325                                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
11326                                        if (relation.getProcess().getGspObject() != null) {
11327                                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
11328                                        }
11329                                }
11330
11331                                if (relation.getPartition() != null) {
11332                                        relationElement.setPartition(relation.getPartition());
11333                                }
11334                                relationElement.setSqlHash(relation.getSqlHash());
11335                                relationElement.setSqlComment(relation.getSqlComment());
11336                                
11337                                if (relation.getProcedureId() != null) {
11338                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
11339                                }
11340                                
11341                                if (relation instanceof JoinRelationship) {
11342                                        relationElement.setCondition(((JoinRelationship) relation).getJoinCondition());
11343                                        relationElement.setJoinType(((JoinRelationship) relation).getJoinType().name());
11344                                        relationElement.setClause(((JoinRelationship) relation).getJoinClauseType().name());
11345                                }
11346
11347                                if(relation instanceof ImpactRelationship){
11348                                        ImpactRelationship impactRelationship = (ImpactRelationship)relation;
11349                                        if(impactRelationship.getJoinClauseType()!=null){
11350                                                relationElement.setClause(impactRelationship.getJoinClauseType().name());
11351                                        }
11352                                }
11353
11354                                String targetName = null;
11355                                Object columnObject = null;
11356                                List<TObjectName> targetObjectNames = null;
11357
11358                                if (targetElement instanceof ResultSetRelationRows) {
11359                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
11360                                        targetColumn target = new targetColumn();
11361                                        target.setId(String.valueOf(targetColumn.getId()));
11362                                        target.setColumn(targetColumn.getName());
11363                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
11364                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
11365                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11366                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11367                                                                + convertCoordinate(targetColumn.getEndPosition()));
11368                                        }
11369                                        if (relation instanceof RecordSetRelationship) {
11370                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11371                                        }
11372                                        target.setSource("system");
11373                                        targetName = targetColumn.getName();
11374                                        relationElement.setTarget(target);
11375                                } else if (targetElement instanceof TableRelationRows) {
11376                                        TableRelationRows targetColumn = (TableRelationRows) targetElement;
11377                                        targetColumn target = new targetColumn();
11378                                        target.setId(String.valueOf(targetColumn.getId()));
11379                                        target.setColumn(targetColumn.getName());
11380                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
11381                                        target.setParent_name(getTableName(targetColumn.getHolder()));
11382                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11383                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11384                                                                + convertCoordinate(targetColumn.getEndPosition()));
11385                                        }
11386                                        if (relation instanceof RecordSetRelationship) {
11387                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11388                                        }
11389                                        target.setSource("system");
11390                                        targetName = targetColumn.getName();
11391                                        relationElement.setTarget(target);
11392                                } else if (targetElement instanceof ResultColumn) {
11393                                        ResultColumn targetColumn = (ResultColumn) targetElement;
11394                                        targetColumn target = new targetColumn();
11395                                        target.setId(String.valueOf(targetColumn.getId()));
11396                                        target.setColumn(targetColumn.getName());
11397                                        target.setStruct(targetColumn.isStruct());
11398                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
11399                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
11400                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11401                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11402                                                                + convertCoordinate(targetColumn.getEndPosition()));
11403                                        }
11404                                        if (relation instanceof RecordSetRelationship) {
11405                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11406                                        }
11407                                        targetName = targetColumn.getName();
11408                                        if (((ResultColumn) targetColumn).getColumnObject() instanceof TResultColumn) {
11409                                                columnsInExpr visitor = new columnsInExpr();
11410                                                ((TResultColumn) ((ResultColumn) targetColumn).getColumnObject()).getExpr()
11411                                                                .inOrderTraverse(visitor);
11412                                                targetObjectNames = visitor.getObjectNames();
11413                                        }
11414
11415                                        if (targetElement instanceof FunctionResultColumn) {
11416                                                columnObject = ((FunctionResultColumn) targetElement).getColumnObject();
11417                                        }
11418                                        if(targetColumn.isPseduo()) {
11419                                                target.setSource("system");
11420                                        }
11421                                        relationElement.setTarget(target);
11422                                } else if (targetElement instanceof TableColumn) {
11423                                        TableColumn targetColumn = (TableColumn) targetElement;
11424                                        targetColumn target = new targetColumn();
11425                                        target.setId(String.valueOf(targetColumn.getId()));
11426                                        target.setColumn(targetColumn.getName());
11427                                        target.setStruct(targetColumn.isStruct());
11428                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
11429                                        target.setParent_name(getTableName(targetColumn.getTable()));
11430                                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
11431                                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
11432                                        }
11433                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11434                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11435                                                                + convertCoordinate(targetColumn.getEndPosition()));
11436                                        }
11437                                        if (relation instanceof RecordSetRelationship) {
11438                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11439                                        }
11440                                        if(targetColumn.isPseduo()) {
11441                                                target.setSource("system");
11442                                        }
11443                                        targetName = targetColumn.getName();
11444                                        relationElement.setTarget(target);
11445                                } else if (targetElement instanceof Argument) {
11446                                        Argument targetColumn = (Argument) targetElement;
11447                                        targetColumn target = new targetColumn();
11448                                        target.setId(String.valueOf(targetColumn.getId()));
11449                                        target.setColumn(targetColumn.getName());
11450                                        target.setParent_id(String.valueOf(targetColumn.getProcedure().getId()));
11451                                        target.setParent_name(targetColumn.getProcedure().getName());
11452                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11453                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11454                                                                + convertCoordinate(targetColumn.getEndPosition()));
11455                                        }
11456                                        if (relation instanceof RecordSetRelationship) {
11457                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11458                                        }
11459                                        targetName = targetColumn.getName();
11460                                        relationElement.setTarget(target);
11461                                } else if (targetElement instanceof Table) {
11462                                        Table table = (Table) targetElement;
11463                                        targetColumn target = new targetColumn();
11464                                        target.setTarget_id(String.valueOf(table.getId()));
11465                                        target.setTarget_name(getTableName(table));
11466                                        if (table.getStartPosition() != null && table.getEndPosition() != null) {
11467                                                target.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
11468                                                                + convertCoordinate(table.getEndPosition()));
11469                                        }
11470                                        relationElement.setTarget(target);
11471                                } else {
11472                                        continue;
11473                                }
11474
11475                                Collection<RelationshipElement<?>> sourceElements = relation.getSources();
11476                                if (sourceElements.size() == 0) {
11477                                        if(clazz == CrudRelationship.class && option.getAnalyzeMode() == AnalyzeMode.crud) {
11478                                                dataflow.getRelationships().add(relationElement);
11479                                        }
11480                                        continue;
11481                                }
11482
11483                                boolean append = false;
11484                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
11485                                        Object sourceElement = sourceItem.getElement();
11486                                        TObjectName sourceColumnName = null;
11487                                        if (sourceItem instanceof ResultColumnRelationshipElement) {
11488                                                sourceColumnName = ((ResultColumnRelationshipElement) sourceItem).getColumnName();
11489                                        }
11490//                                      if (sourceItem instanceof TableColumnRelationElement
11491//                                                      && (((TableColumnRelationElement) sourceItem).getColumnIndex() != null)) {
11492//                                              TableColumnRelationElement tableColumnRelationElement = (TableColumnRelationElement) sourceItem;
11493//                                              sourceElement = tableColumnRelationElement.getElement().getTable().getColumns()
11494//                                                              .get(tableColumnRelationElement.getColumnIndex() + 1);
11495//                                      }
11496                                        if (sourceElement instanceof ResultColumn) {
11497                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
11498                                                if (sourceColumn.hasStarLinkColumn() && !relation.isShowStarRelation() && !sourceColumn.isPseduo()) {
11499                                                        List<String> sourceStarColumnNames = sourceColumn.getStarLinkColumnNames();
11500                                                        sourceColumn source = new sourceColumn();
11501                                                        if (targetObjectNames != null && !targetObjectNames.isEmpty()) {
11502
11503                                                                for (int k = 0; k < targetObjectNames.size(); k++) {
11504                                                                        String targetObjectName = getColumnName(targetObjectNames.get(k));
11505                                                                        if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
11506                                                                                boolean find = false;
11507                                                                                for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
11508                                                                                        if(column.getName().equalsIgnoreCase(targetObjectName)) {
11509                                                                                                source = new sourceColumn();
11510                                                                                                source.setId(String.valueOf(column.getId()));
11511                                                                                                source.setColumn(targetObjectName);
11512                                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11513                                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11514                                                                                                if (sourceColumn.getStartPosition() != null
11515                                                                                                                && sourceColumn.getEndPosition() != null) {
11516                                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11517                                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
11518                                                                                                }
11519                                                                                                append = true;
11520                                                                                                relationElement.addSource(source);
11521                                                                                                find = true;
11522                                                                                                break;
11523                                                                                        }
11524                                                                                }
11525                                                                                if(find) {
11526                                                                                        continue;
11527                                                                                }
11528                                                                        }
11529                                                                        if (sourceColumn.getStarLinkColumns().containsKey(targetObjectName)) {
11530                                                                                source = new sourceColumn();
11531                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
11532                                                                                                + sourceStarColumnNames.indexOf(targetObjectName));
11533                                                                                source.setColumn(targetObjectName);
11534                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11535                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11536                                                                                if (sourceColumn.getStartPosition() != null
11537                                                                                                && sourceColumn.getEndPosition() != null) {
11538                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11539                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
11540                                                                                }
11541                                                                                append = true;
11542                                                                                relationElement.addSource(source);
11543                                                                        } else {
11544                                                                                source = new sourceColumn();
11545                                                                                source.setId(String.valueOf(sourceColumn.getId()));
11546                                                                                source.setColumn(relationElement.getTarget().getColumn());
11547                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11548                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11549                                                                                if (sourceColumn.getStartPosition() != null
11550                                                                                                && sourceColumn.getEndPosition() != null) {
11551                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11552                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
11553                                                                                }
11554                                                                                append = true;
11555                                                                                relationElement.addSource(source);
11556                                                                        }
11557                                                                }
11558                                                        } else {
11559                                                                if (columnObject instanceof TWhenClauseItemList) {
11560                                                                        TCaseExpression expr = (TCaseExpression)((FunctionResultColumn) targetElement).getResultSet().getGspObject();
11561                                                                        List<TExpression> directExpressions = new ArrayList<TExpression>();
11562                                                                        
11563//                                                                      TExpression inputExpr = expr.getInput_expr();
11564//                                                                      if (inputExpr != null) {
11565//                                                                              directExpressions.add(inputExpr);
11566//                                                                      }
11567                                                                        TExpression defaultExpr = expr.getElse_expr();
11568                                                                        if (defaultExpr != null) {
11569                                                                                directExpressions.add(defaultExpr);
11570                                                                        }
11571                                                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
11572                                                                        for (int k = 0; k < list.size(); k++) {
11573                                                                                TWhenClauseItem element = list.getWhenClauseItem(k);
11574                                                                                directExpressions.add(element.getReturn_expr());
11575                                                                        }
11576                                                                        
11577                                                                        for (int k = 0; k < directExpressions.size(); k++) {
11578                                                                                columnsInExpr visitor = new columnsInExpr();
11579                                                                                directExpressions.get(k).inOrderTraverse(visitor);
11580                                                                                List<TObjectName> objectNames = visitor.getObjectNames();
11581                                                                                if (objectNames == null) {
11582                                                                                        continue;
11583                                                                                }
11584                                                                                for (int x = 0; x < objectNames.size(); x++) {
11585                                                                                        String objectName = getColumnName(objectNames.get(x));
11586                                                                                        if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
11587
11588                                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
11589                                                                                                        boolean find = false;
11590                                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
11591                                                                                                                if(column.getName().equalsIgnoreCase(objectName)) {
11592                                                                                                                        source = new sourceColumn();
11593                                                                                                                        source.setId(String.valueOf(column.getId()));
11594                                                                                                                        source.setColumn(objectName);
11595                                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11596                                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11597                                                                                                                        if (sourceColumn.getStartPosition() != null
11598                                                                                                                                        && sourceColumn.getEndPosition() != null) {
11599                                                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11600                                                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
11601                                                                                                                        }
11602                                                                                                                        append = true;
11603                                                                                                                        relationElement.addSource(source);
11604                                                                                                                        find = true;
11605                                                                                                                        break;
11606                                                                                                                }
11607                                                                                                        }
11608                                                                                                        if(find) {
11609                                                                                                                continue;
11610                                                                                                        }
11611                                                                                                }
11612                                                                                                
11613                                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
11614                                                                                                                + sourceStarColumnNames.indexOf(objectName));
11615                                                                                                source.setColumn(objectName);
11616                                                                                        } else {
11617                                                                                                source.setId(String.valueOf(sourceColumn.getId()));
11618                                                                                                source.setColumn(sourceColumn.getName());
11619                                                                                        }
11620                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11621                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11622                                                                                        if (sourceColumn.getStartPosition() != null
11623                                                                                                        && sourceColumn.getEndPosition() != null) {
11624                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11625                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
11626                                                                                        }
11627                                                                                        append = true;
11628                                                                                        relationElement.addSource(source);
11629                                                                                }
11630                                                                        }
11631                                                                } else {
11632                                                                        String objectName = getColumnName(targetName);
11633                                                                        boolean find = false;
11634                                                                        
11635                                                                        if (!find && sourceColumn.getStarLinkColumns().containsKey(objectName)) {
11636                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
11637                                                                                                + sourceStarColumnNames.indexOf(objectName));
11638                                                                                source.setColumn(objectName);
11639                                                                                find = true;
11640                                                                        }
11641
11642                                                                        if (!find && sourceColumnName != null) {
11643                                                                                objectName = getColumnName(sourceColumnName);
11644                                                                                
11645                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
11646                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
11647                                                                                                if(column.getName().equalsIgnoreCase(objectName)) {
11648                                                                                                        source = new sourceColumn();
11649                                                                                                        source.setId(String.valueOf(column.getId()));
11650                                                                                                        source.setColumn(objectName);
11651                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11652                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11653                                                                                                        if (sourceColumn.getStartPosition() != null
11654                                                                                                                        && sourceColumn.getEndPosition() != null) {
11655                                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11656                                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
11657                                                                                                        }
11658                                                                                                        append = true;
11659                                                                                                        relationElement.addSource(source);
11660                                                                                                        find = true;
11661                                                                                                        break;
11662                                                                                                }
11663                                                                                        }
11664                                                                                        if(find) {
11665                                                                                                continue;
11666                                                                                        }
11667                                                                                }
11668                                                                                
11669                                                                                if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
11670                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
11671                                                                                                        + sourceStarColumnNames.indexOf(objectName));
11672                                                                                        source.setColumn(objectName);
11673                                                                                        find = true;
11674                                                                                }
11675                                                                        }
11676                                                                        
11677                                                                        if (!find && sourceItem instanceof ResultColumnRelationshipElement) {
11678                                                                                int starIndex = ((ResultColumnRelationshipElement) sourceItem)
11679                                                                                                .getStarIndex();
11680                                                                                if (starIndex > -1 && sourceColumn.getStarLinkColumnList().size() > starIndex) {
11681                                                                                        objectName = getColumnName(sourceColumn.getStarLinkColumnList().get(starIndex));
11682                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
11683                                                                                                        + starIndex);
11684                                                                                        source.setColumn(objectName);
11685                                                                                        find = true;
11686                                                                                }
11687                                                                        }
11688
11689                                                                        if (!find) {
11690                                                                                source.setId(String.valueOf(sourceColumn.getId()));
11691                                                                                source.setColumn(sourceColumn.getName());
11692                                                                        }
11693
11694                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11695                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11696                                                                        if (sourceColumn.getStartPosition() != null
11697                                                                                        && sourceColumn.getEndPosition() != null) {
11698                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11699                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11700                                                                        }
11701                                                                        append = true;
11702                                                                        relationElement.addSource(source);
11703                                                                }
11704                                                        }
11705
11706                                                } else {
11707                                                        sourceColumn source = new sourceColumn();
11708                                                        source.setId(String.valueOf(sourceColumn.getId()));
11709                                                        source.setColumn(sourceColumn.getName());
11710                                                        source.setStruct(sourceColumn.isStruct());
11711                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11712                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11713                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11714                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11715                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11716                                                        }
11717                                                        append = true;
11718                                                        if (sourceItem.getTransforms() != null) {
11719                                                                for (Transform transform : sourceItem.getTransforms()) {
11720                                                                        source.addTransform(transform);
11721                                                                }
11722                                                        }
11723                                                        if(sourceColumn.isPseduo()) {
11724                                                                source.setSource("system");
11725                                                        }
11726                                                        relationElement.addSource(source);
11727                                                }
11728                                        } else if (sourceElement instanceof TableColumn) {
11729                                                TableColumn sourceColumn = (TableColumn) sourceElement;
11730                                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()
11731                                                                && sourceItem instanceof TableColumnRelationshipElement) {
11732                                                        sourceColumn source = new sourceColumn();
11733                                                        boolean find = false;
11734                                                        if (((TableColumnRelationshipElement) sourceItem).getColumnIndex() != null) {
11735                                                                int columnIndex = ((TableColumnRelationshipElement) sourceItem).getColumnIndex();
11736                                                                if (sourceColumn.getStarLinkColumns().size() > columnIndex) {
11737                                                                        source = new sourceColumn();
11738                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + columnIndex);
11739                                                                        String targetObjectName = getColumnName(
11740                                                                                        sourceColumn.getStarLinkColumnList().get(columnIndex));
11741                                                                        source.setColumn(targetObjectName);
11742                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
11743                                                                        source.setParent_name(sourceColumn.getTable().getName());
11744                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
11745                                                                                source.setParent_alias(
11746                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
11747                                                                        }
11748                                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11749                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11750                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11751                                                                        }
11752                                                                        append = true;
11753                                                                        relationElement.addSource(source);
11754                                                                        find = true;
11755                                                                }
11756                                                        }
11757
11758                                                        if (!find) {
11759                                                                String objectName = getColumnName(targetName);
11760
11761                                                                table tableElement = null;
11762                                                                if (dataflow.getTables() != null) {
11763                                                                        for (table t : dataflow.getTables()) {
11764                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
11765                                                                                        tableElement = t;
11766                                                                                        break;
11767                                                                                }
11768                                                                        }
11769                                                                }
11770                                                                if (tableElement == null && dataflow.getViews() != null) {
11771                                                                        for (table t : dataflow.getViews()) {
11772                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
11773                                                                                        tableElement = t;
11774                                                                                        break;
11775                                                                                }
11776                                                                        }
11777                                                                }
11778                                                                if (tableElement == null && dataflow.getVariables() != null) {
11779                                                                        for (table t : dataflow.getVariables()) {
11780                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
11781                                                                                        tableElement = t;
11782                                                                                        break;
11783                                                                                }
11784                                                                        }
11785                                                                }
11786
11787                                                                if(tableElement!=null) {
11788                                                                        for (column column : tableElement.getColumns()) {
11789                                                                                if (column.getName().equalsIgnoreCase(objectName)) {
11790                                                                                        source = new sourceColumn();
11791                                                                                        source.setId(String.valueOf(column.getId()));
11792                                                                                        source.setColumn(objectName);
11793                                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
11794                                                                                        source.setParent_name(sourceColumn.getTable().getName());
11795                                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
11796                                                                                                source.setParent_alias(
11797                                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
11798                                                                                        }
11799                                                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11800                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11801                                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11802                                                                                        }
11803                                                                                        if (sourceItem.getTransforms() != null) {
11804                                                                                                for (Transform transform : sourceItem.getTransforms()) {
11805                                                                                                        source.addTransform(transform);
11806                                                                                                }
11807                                                                                        }
11808                                                                                        if (sourceColumn.getCandidateParents() != null) {
11809                                                                                                for(Object item: sourceColumn.getCandidateParents()) {
11810                                                                                                        candidateTable candidateParent = new candidateTable();
11811                                                                                                        if(item instanceof Table) {
11812                                                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
11813                                                                                                                candidateParent.setName(getTableName((Table)item));
11814                                                                                                                source.addCandidateParent(candidateParent);
11815                                                                                                        }
11816                                                                                                        else if(item instanceof ResultSet) {
11817                                                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
11818                                                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
11819                                                                                                                source.addCandidateParent(candidateParent);
11820                                                                                                        }
11821                                                                                                }
11822                                                                                        }
11823                                                                                        append = true;
11824                                                                                        relationElement.addSource(source);
11825                                                                                        find = true;
11826                                                                                        break;
11827                                                                                }
11828                                                                        }
11829                                                                }
11830                                                        }
11831
11832                                                        if(!find) {
11833                                                                source = new sourceColumn();
11834                                                                source.setId(String.valueOf(sourceColumn.getId()));
11835                                                                source.setColumn(sourceColumn.getName());
11836                                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
11837                                                                source.setParent_name(sourceColumn.getTable().getName());
11838                                                                if (sourceItem instanceof TableColumnRelationshipElement) {
11839                                                                        source.setParent_alias(
11840                                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
11841                                                                }
11842                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11843                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11844                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
11845                                                                }
11846                                                                if (sourceItem.getTransforms() != null) {
11847                                                                        for (Transform transform : sourceItem.getTransforms()) {
11848                                                                                source.addTransform(transform);
11849                                                                        }
11850                                                                }
11851                                                                if (sourceColumn.getCandidateParents() != null) {
11852                                                                        for(Object item: sourceColumn.getCandidateParents()) {
11853                                                                                candidateTable candidateParent = new candidateTable();
11854                                                                                if(item instanceof Table) {
11855                                                                                        candidateParent.setId(String.valueOf(((Table)item).getId()));
11856                                                                                        candidateParent.setName(getTableName((Table)item));
11857                                                                                        source.addCandidateParent(candidateParent);
11858                                                                                }
11859                                                                                else if(item instanceof ResultSet) {
11860                                                                                        candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
11861                                                                                        candidateParent.setName(getResultSetName((ResultSet)item));
11862                                                                                        source.addCandidateParent(candidateParent);
11863                                                                                }
11864                                                                        }
11865                                                                }
11866                                                                append = true;
11867                                                                relationElement.addSource(source);
11868                                                        }
11869
11870                                                } else {
11871                                                        sourceColumn source = new sourceColumn();
11872                                                        source.setId(String.valueOf(sourceColumn.getId()));
11873                                                        source.setColumn(sourceColumn.getName());
11874                                                        source.setStruct(sourceColumn.isStruct());
11875                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
11876                                                        source.setParent_name(getTableName(sourceColumn.getTable()));
11877                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
11878                                                                source.setParent_alias(
11879                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
11880                                                        }
11881                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11882                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11883                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11884                                                        }
11885                                                        if (sourceItem.getTransforms() != null) {
11886                                                                for (Transform transform : sourceItem.getTransforms()) {
11887                                                                        source.addTransform(transform);
11888                                                                }
11889                                                        }
11890                                                        if(sourceColumn.isPseduo()) {
11891                                                                source.setSource("system");
11892                                                        }
11893                                                        if (sourceColumn.getCandidateParents() != null) {
11894                                                                for(Object item: sourceColumn.getCandidateParents()) {
11895                                                                        candidateTable candidateParent = new candidateTable();
11896                                                                        if(item instanceof Table) {
11897                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
11898                                                                                candidateParent.setName(getTableName((Table)item));
11899                                                                                source.addCandidateParent(candidateParent);
11900                                                                        }
11901                                                                        else if(item instanceof ResultSet) {
11902                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
11903                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
11904                                                                                source.addCandidateParent(candidateParent);
11905                                                                        }
11906                                                                }
11907                                                        }
11908                                                        append = true;
11909                                                        relationElement.addSource(source);
11910                                                }
11911                                        } else if (sourceElement instanceof Argument) {
11912                                                Argument sourceColumn = (Argument) sourceElement;
11913                                                sourceColumn source = new sourceColumn();
11914                                                source.setId(String.valueOf(sourceColumn.getId()));
11915                                                source.setColumn(sourceColumn.getName());
11916                                                source.setParent_id(String.valueOf(sourceColumn.getProcedure().getId()));
11917                                                source.setParent_name(sourceColumn.getProcedure().getName());
11918                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11919                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11920                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
11921                                                }
11922                                                append = true;
11923                                                relationElement.addSource(source);
11924                                        } else if (sourceElement instanceof TableRelationRows) {
11925                                                TableRelationRows sourceColumn = (TableRelationRows) sourceElement;
11926                                                sourceColumn source = new sourceColumn();
11927                                                source.setId(String.valueOf(sourceColumn.getId()));
11928                                                source.setColumn(sourceColumn.getName());
11929                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
11930                                                source.setParent_name(getTableName(sourceColumn.getHolder()));
11931                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11932                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11933                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
11934                                                }
11935                                                source.setSource("system");
11936                                                append = true;
11937                                                relationElement.addSource(source);
11938                                        } else if (sourceElement instanceof ResultSetRelationRows) {
11939                                                ResultSetRelationRows sourceColumn = (ResultSetRelationRows) sourceElement;
11940                                                sourceColumn source = new sourceColumn();
11941                                                source.setId(String.valueOf(sourceColumn.getId()));
11942                                                source.setColumn(sourceColumn.getName());
11943                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
11944                                                source.setParent_name(getResultSetName(sourceColumn.getHolder()));
11945                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11946                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11947                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
11948                                                }
11949                                                source.setSource("system");
11950                                                append = true;
11951                                                relationElement.addSource(source);
11952                                        } else if (sourceElement instanceof Constant) {
11953                                                Constant sourceColumn = (Constant) sourceElement;
11954                                                sourceColumn source = new sourceColumn();
11955                                                source.setId(String.valueOf(sourceColumn.getId()));
11956                                                source.setColumn(sourceColumn.getName());
11957                                                source.setColumn_type("constant");
11958                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11959                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11960                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
11961                                                }
11962                                                append = true;
11963                                                relationElement.addSource(source);
11964                                        } else if (sourceElement instanceof Table) {
11965                                                Table table = (Table) sourceElement;
11966                                                sourceColumn source = new sourceColumn();
11967                                                source.setSource_id(String.valueOf(table.getId()));
11968                                                source.setSource_name(getTableName(table));
11969                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
11970                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
11971                                                                        + convertCoordinate(table.getEndPosition()));
11972                                                }
11973                                                append = true;
11974                                                relationElement.addSource(source);
11975                                        }
11976
11977                                        if (relation instanceof ImpactRelationship) {
11978                                                ESqlClause clause = getSqlClause(sourceItem);
11979                                                if (clause != null
11980                                                                && (relationElement.getSources() != null && !relationElement.getSources().isEmpty())) {
11981                                                        relationElement.getSources().get(relationElement.getSources().size() - 1)
11982                                                                        .setClauseType(clause.name());
11983                                                }
11984                                        }
11985                                }
11986                                if (append)
11987                                        dataflow.getRelationships().add(relationElement);
11988                        }
11989                }
11990        }
11991
11992        private boolean containsStar(Collection<RelationshipElement<?>> elements) {
11993                if (elements == null || elements.size() == 0)
11994                        return false;
11995
11996                for (RelationshipElement<?> element : elements) {
11997                        if (element.getElement() instanceof ResultColumn) {
11998                                ResultColumn object = (ResultColumn) element.getElement();
11999                                if (object.getName().endsWith("*") && object.isShowStar())
12000                                        return true;
12001                        } else if (element.getElement() instanceof TableColumn) {
12002                                TableColumn object = (TableColumn) element.getElement();
12003                                if (object.getName().endsWith("*") && object.isShowStar())
12004                                        return true;
12005                        }
12006                }
12007                return false;
12008        }
12009
12010        private Map<String, Set<String>> appendTableStarColumns = new HashMap<String, Set<String>>();
12011
12012        // Tracks column names that have explicit (non-star) sources per target star column.
12013        // Used across relationships to prevent phantom star expansions.
12014        // Key: target star column ID, Value: set of column names with explicit sources.
12015        private Map<Long, Set<String>> explicitStarTargetColumns = new HashMap<Long, Set<String>>();
12016        private void updateResultColumnStarLinks(dataflow dataflow, AbstractRelationship relation, int index) {
12017        try {
12018            if (option.getAnalyzeMode() == AnalyzeMode.crud) {
12019                return;
12020            }
12021
12022            ResultColumn targetColumn = (ResultColumn) relation.getTarget().getElement();
12023
12024            Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
12025            if (sourceElements == null || sourceElements.size() == 0)
12026                return;
12027
12028            for (RelationshipElement<?> sourceItem : sourceElements) {
12029                Object sourceElement = sourceItem.getElement();
12030                if (sourceElement instanceof ResultColumn) {
12031                    ResultColumn source = (ResultColumn) sourceElement;
12032                    if (source.hasStarLinkColumn()) {
12033                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12034                            if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12035                                targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12036                            }
12037                            targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12038                        }
12039                        if (!source.isShowStar()) {
12040                            targetColumn.setShowStar(false);
12041                            relation.setShowStarRelation(false);
12042                        }
12043                    } else if (!"*".equals(source.getName())) {
12044                        if (source.getColumnObject() instanceof TObjectName) {
12045                            if (source instanceof FunctionResultColumn) {
12046
12047                            } else {
12048                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
12049                            }
12050                        } else if (source.getColumnObject() instanceof TResultColumn) {
12051                            TResultColumn sourceColumn = (TResultColumn) source.getColumnObject();
12052                            if (sourceColumn.getAliasClause() != null) {
12053                                targetColumn.bindStarLinkColumn(sourceColumn.getAliasClause().getAliasName());
12054                            } else if (sourceColumn.getFieldAttr() != null) {
12055                                targetColumn.bindStarLinkColumn(sourceColumn.getFieldAttr());
12056                            } else {
12057                                TObjectName column = new TObjectName();
12058                                if (sourceColumn.getExpr().getExpressionType() == EExpressionType.typecast_t) {
12059                                    column.setString(sourceColumn.getExpr().getLeftOperand().toString());
12060                                } else {
12061                                    column.setString(sourceColumn.toString());
12062                                }
12063                                targetColumn.bindStarLinkColumn(column);
12064                            }
12065                        }
12066                    }
12067                } else if (sourceElement instanceof TableColumn) {
12068                    TableColumn source = (TableColumn) sourceElement;
12069                    if (!source.isPseduo() && source.hasStarLinkColumn()) {
12070                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12071                            if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12072                                targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12073                            }
12074                            targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12075                        }
12076                        if ((source.getTable().isCreateTable() || source.getTable().hasSQLEnv()) && !source.isShowStar()) {
12077                            targetColumn.setShowStar(false);
12078                            relation.setShowStarRelation(false);
12079                        }
12080
12081                    } else if (!"*".equals(source.getName())) {
12082                        targetColumn.bindStarLinkColumn(source.getColumnObject());
12083                    }
12084                }
12085            }
12086
12087            if (targetColumn.hasStarLinkColumn()) {
12088                table resultSetElement = null;
12089                for (table t : dataflow.getResultsets()) {
12090                    if (t.getId().equals(String.valueOf(targetColumn.getResultSet().getId()))) {
12091                        resultSetElement = t;
12092                        break;
12093                    }
12094                }
12095
12096                int starColumnCount = 0;
12097                for (column item : resultSetElement.getColumns()) {
12098                    if (item.getName() != null && item.getName().endsWith("*")) {
12099                        starColumnCount += 1;
12100                    }
12101                }
12102
12103                if (resultSetElement != null && starColumnCount <= 1) {
12104                    List<String> columns = targetColumn.getStarLinkColumnNames();
12105                    if (index == -1) {
12106                        for (int k = 0; k < columns.size(); k++) {
12107                            String columnName = columns.get(k);
12108                            String id = String.valueOf(targetColumn.getId()) + "_" + k;
12109                            if (appendTableStarColumns.containsKey(resultSetElement.getId()) && appendTableStarColumns.get(resultSetElement.getId()).contains(id)) {
12110                                continue;
12111                            }
12112                            column columnElement = new column();
12113                            columnElement.setId(id);
12114                            columnElement.setName(columnName);
12115                            if (targetColumn.isFunction()) {
12116                                columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
12117                            }
12118                            if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12119                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12120                                        + convertCoordinate(targetColumn.getEndPosition()));
12121                            }
12122
12123                            if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
12124                                boolean find = false;
12125                                for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12126                                    ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12127                                    if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName())
12128                                            .equals(columnName)) {
12129                                        find = true;
12130                                        break;
12131                                    }
12132                                }
12133                                if (find) {
12134                                    continue;
12135                                }
12136                            }
12137
12138                            if (!resultSetElement.getColumns().contains(columnElement)) {
12139                                resultSetElement.getColumns().add(columnElement);
12140                                appendTableStarColumns.putIfAbsent(resultSetElement.getId(), new HashSet<String>());
12141                                appendTableStarColumns.get(resultSetElement.getId()).add(id);
12142                            }
12143                        }
12144                    } else {
12145                        int k = index;
12146                        String columnName = columns.get(k);
12147                        column columnElement = new column();
12148                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
12149                        columnElement.setName(columnName);
12150                        if (targetColumn.isFunction()) {
12151                            columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
12152                        }
12153                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12154                            columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12155                                    + convertCoordinate(targetColumn.getEndPosition()));
12156                        }
12157
12158                        if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
12159                            boolean find = false;
12160                            for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12161                                ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12162                                if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(columnName)) {
12163                                    find = true;
12164                                    break;
12165                                }
12166                            }
12167                            if (find) {
12168                                return;
12169                            }
12170                        }
12171
12172                        if (!resultSetElement.getColumns().contains(columnElement)) {
12173                            resultSetElement.getColumns().add(columnElement);
12174                        }
12175                    }
12176                }
12177            }
12178        } catch (Exception e) {
12179            logger.error("updateResultColumnStarLinks occurs unknown exceptions.", e);
12180        }
12181        }
12182
12183        private void updateTableColumnStarLinks(dataflow dataflow, AbstractRelationship relation) {
12184                TableColumn targetColumn = (TableColumn) relation.getTarget().getElement();             
12185                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
12186                if (sourceElements == null || sourceElements.size() == 0)
12187                        return;
12188
12189                TableColumn sourceStarColumn = null;
12190
12191                for (RelationshipElement<?> sourceItem: sourceElements) {
12192                        Object sourceElement = sourceItem.getElement();
12193                        if (sourceElement instanceof ResultColumn) {
12194                                ResultColumn source = (ResultColumn) sourceElement;
12195                                if(source.getResultSet() instanceof Function) {
12196                                        continue;
12197                                }
12198                                if (source.hasStarLinkColumn()) {
12199                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12200                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12201                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12202                                                }
12203                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12204                                        }
12205                                        if (!source.isShowStar()) {
12206                                                targetColumn.setShowStar(false);
12207                                                relation.setShowStarRelation(false);
12208                                        }
12209                                } else if (!"*".equals(source.getName())) {
12210                                        if (source.getColumnObject() instanceof TObjectName) {
12211                                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
12212                                        } else if (source.getColumnObject() instanceof TResultColumn) {
12213                                                if (((TResultColumn) source.getColumnObject()).getAliasClause() != null) {
12214                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getAliasClause()
12215                                                                        .getAliasName();
12216                                                        if (field != null) {
12217                                                                targetColumn.bindStarLinkColumn(field);
12218                                                        }
12219                                                } else {
12220                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getFieldAttr();
12221                                                        if (field != null) {
12222                                                                targetColumn.bindStarLinkColumn(field);
12223                                                        } else {
12224                                                                TObjectName column = new TObjectName();
12225                                                                if (((TResultColumn) source.getColumnObject()).getExpr()
12226                                                                                .getExpressionType() == EExpressionType.typecast_t) {
12227                                                                        column.setString(((TResultColumn) source.getColumnObject()).getExpr()
12228                                                                                        .getLeftOperand().toString());
12229                                                                } else {
12230                                                                        column.setString(((TResultColumn) source.getColumnObject()).toString());
12231                                                                }
12232                                                                targetColumn.bindStarLinkColumn(column);
12233                                                        }
12234                                                }
12235                                        }
12236                                }
12237                        } else if (sourceElement instanceof TableColumn) {
12238                                TableColumn source = (TableColumn) sourceElement;
12239                                if (source.hasStarLinkColumn()) {
12240                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12241                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12242                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12243                                                }
12244                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12245                                        }
12246                                        for (Map.Entry<String, Set<TObjectName>> item : targetColumn.getStarLinkColumns().entrySet()) {
12247                                                if (!source.getStarLinkColumns().containsKey(item.getKey())) {
12248                                                        source.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12249                                                }
12250                                                source.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12251                                        }
12252
12253                                        sourceStarColumn = source;
12254
12255                                        if (source.getTable().isCreateTable() && !source.isShowStar()) {
12256                                                targetColumn.setShowStar(false);
12257                                                relation.setShowStarRelation(false);
12258                                        }
12259                                } else if (!"*".equals(source.getName())) {
12260                                        if (source.isStruct()) {
12261                                                targetColumn.bindStarLinkColumn(source.getColumnObject());
12262                                        }
12263                                        else {
12264                                                TObjectName objectName = new TObjectName();
12265                                                objectName.setString(DlineageUtil.getColumnNameOnly(source.getName()));
12266                                                targetColumn.bindStarLinkColumn(objectName);
12267                                        }
12268                                }
12269                        }
12270                }
12271
12272                if (targetColumn.hasStarLinkColumn()) {
12273                        table tableElement = null;
12274                        if (dataflow.getTables() != null) {
12275                                for (table t : dataflow.getTables()) {
12276                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
12277                                                tableElement = t;
12278                                                break;
12279                                        }
12280                                }
12281                        }
12282                        if (tableElement == null && dataflow.getViews() != null) {
12283                                for (table t : dataflow.getViews()) {
12284                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
12285                                                tableElement = t;
12286                                                break;
12287                                        }
12288                                }
12289                        }
12290                        if (tableElement == null && dataflow.getVariables() != null) {
12291                                for (table t : dataflow.getVariables()) {
12292                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
12293                                                tableElement = t;
12294                                                break;
12295                                        }
12296                                }
12297                        }
12298
12299                        if (tableElement != null) {
12300                                List<String> columns = new ArrayList<String>(targetColumn.getStarLinkColumns().keySet());
12301                                for (int k = 0; k < columns.size(); k++) {
12302                                        String columnName = columns.get(k);
12303                                        if (containStarColumn(targetColumn.getTable().getColumns(), columnName)) {
12304                                                continue;
12305                                        }
12306                                        column columnElement = new column();
12307                                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
12308                                        columnElement.setName(columnName);
12309                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12310                                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12311                                                                + convertCoordinate(targetColumn.getEndPosition()));
12312                                        }
12313                                        if (!tableElement.getColumns().contains(columnElement)) {
12314                                                tableElement.getColumns().add(columnElement);
12315                                        }
12316                                }
12317                        }
12318                }
12319
12320                if (sourceStarColumn != null) {
12321                        table tableElement = null;
12322                        if (dataflow.getTables() != null) {
12323                                for (table t : dataflow.getTables()) {
12324                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
12325                                                tableElement = t;
12326                                                break;
12327                                        }
12328                                }
12329                        }
12330                        if (tableElement == null && dataflow.getViews() != null) {
12331                                for (table t : dataflow.getViews()) {
12332                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
12333                                                tableElement = t;
12334                                                break;
12335                                        }
12336                                }
12337                        }
12338                        if (tableElement == null && dataflow.getVariables() != null) {
12339                                for (table t : dataflow.getVariables()) {
12340                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
12341                                                tableElement = t;
12342                                                break;
12343                                        }
12344                                }
12345                        }
12346
12347                        if (tableElement != null) {
12348                                List<String> columns = new ArrayList<String>(sourceStarColumn.getStarLinkColumns().keySet());
12349                                for (int k = 0; k < columns.size(); k++) {
12350                                        String columnName = columns.get(k);
12351                                        if (containStarColumn(sourceStarColumn.getTable().getColumns(), columnName)) {
12352                                                continue;
12353                                        }
12354                                        column columnElement = new column();
12355                                        columnElement.setId(sourceStarColumn.getId() + "_" + k);
12356                                        columnElement.setName(columnName);
12357                                        if (sourceStarColumn.getStartPosition() != null && sourceStarColumn.getEndPosition() != null) {
12358                                                columnElement.setCoordinate(convertCoordinate(sourceStarColumn.getStartPosition()) + ","
12359                                                                + convertCoordinate(sourceStarColumn.getEndPosition()));
12360                                        }
12361                                        if (!tableElement.getColumns().contains(columnElement)) {
12362                                                tableElement.getColumns().add(columnElement);
12363                                        }
12364                                }
12365                        }
12366                }
12367        }
12368
12369        private ESqlClause getSqlClause(RelationshipElement<?> relationshipElement) {
12370                if (relationshipElement instanceof TableColumnRelationshipElement) {
12371                        return ((TableColumnRelationshipElement) relationshipElement).getRelationLocation();
12372                } else if (relationshipElement instanceof ResultColumnRelationshipElement) {
12373                        return ((ResultColumnRelationshipElement) relationshipElement).getRelationLocation();
12374                }
12375                return null;
12376        }
12377
12378        private void appendStarRelation(dataflow dataflow, AbstractRelationship relation, int index) {
12379                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
12380                        return;
12381                }
12382                
12383                Object targetElement = relation.getTarget().getElement();
12384
12385                relationship relationElement = new relationship();
12386                relationElement.setType(relation.getRelationshipType().name());
12387                if (relation.getEffectType() != null) {
12388                        relationElement.setEffectType(relation.getEffectType().name());
12389                }
12390                relationElement.setSqlHash(relation.getSqlHash());
12391                relationElement.setSqlComment(relation.getSqlComment());
12392
12393                if (relation.getProcedureId() != null) {
12394                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12395                }
12396                relationElement.setId(String.valueOf(relation.getId()) + "_" + index);
12397                if (relation.getProcess() != null) {
12398                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
12399                        if (relation.getProcess().getGspObject() != null) {
12400                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
12401                        }
12402                }
12403                if (relation instanceof DataFlowRelationship) {
12404                        relationElement.setSqlHash(((DataFlowRelationship) relation).getSqlHash());
12405                        relationElement.setSqlComment(((DataFlowRelationship) relation).getSqlComment());
12406                
12407                        if (relation.getProcedureId() != null) {
12408                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12409                        }
12410                }
12411                String targetName = "";
12412
12413                if (targetElement instanceof ResultColumn) {
12414                        ResultColumn targetColumn = (ResultColumn) targetElement;
12415
12416                        targetName = targetColumn.getStarLinkColumnNames().get(index);
12417
12418                        
12419                        targetColumn target = new targetColumn();
12420                        target.setId(String.valueOf(targetColumn.getId()) + "_" + index);
12421                        if(targetColumn.getResultSet()!=null && targetColumn.getResultSet().getColumns()!=null) {
12422                                for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12423                                        ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12424                                        if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(targetName)) {
12425                                                target.setId(String.valueOf(columnModel.getId()));
12426                                                break;
12427                                        }
12428                                }
12429                        }
12430                        target.setColumn(targetName);
12431                        target.setStruct(targetColumn.isStruct());
12432                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
12433                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
12434                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12435                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12436                                                + convertCoordinate(targetColumn.getEndPosition()));
12437                        }
12438                        relationElement.setTarget(target);
12439                } else if (targetElement instanceof TableColumn) {
12440                        TableColumn targetColumn = (TableColumn) targetElement;
12441
12442                        targetName = targetColumn.getStarLinkColumnNames().get(index);
12443
12444                        TableColumn tableColumn = searchTableColumn(targetColumn.getTable().getColumns(), targetName);
12445
12446                        targetColumn target = new targetColumn();
12447                        if (tableColumn == null) {
12448                                target.setId(targetColumn.getId() + "_" + index);
12449                        } else {
12450                                target.setId(String.valueOf(tableColumn.getId()));
12451                        }
12452                        target.setStruct(targetColumn.isStruct());
12453                        target.setColumn(targetName);
12454                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
12455                        target.setParent_name(targetColumn.getTable().getName());
12456                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
12457                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
12458                        }
12459                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12460                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12461                                                + convertCoordinate(targetColumn.getEndPosition()));
12462                        }
12463                        relationElement.setTarget(target);
12464                } else {
12465                        return;
12466                }
12467
12468                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
12469                if (sourceElements.size() == 0) {
12470                        return;
12471                }
12472
12473                String targetIdentifierName = DlineageUtil.getIdentifierNormalColumnName(targetName);
12474                if (targetIdentifierName == null) {
12475                        return;
12476                }
12477
12478                // Track explicit (non-star) source columns per target star column to prevent
12479                // phantom star expansions. Skip for UNION targets where branches contribute independently.
12480                long targetColumnId = -1;
12481                boolean isUnionTarget = false;
12482                if (targetElement instanceof ResultColumn) {
12483                        ResultColumn tc = (ResultColumn) targetElement;
12484                        targetColumnId = tc.getId();
12485                        isUnionTarget = (tc.getResultSet() instanceof SelectSetResultSet);
12486                } else if (targetElement instanceof TableColumn) {
12487                        targetColumnId = ((TableColumn) targetElement).getId();
12488                }
12489
12490                if (!isUnionTarget && targetColumnId != -1) {
12491                        // Check if any source in this relationship is a non-star source matching the target
12492                        // column name, AND whose parent does NOT also contribute a star source.
12493                        // This distinguishes:
12494                        //   - "b.col_a" (definitive: b only provides explicit cols, not *) => suppress star expansion
12495                        //   - "aTab.id" (not definitive: aTab also provides *) => don't suppress
12496                        boolean hasDefinitiveExplicitSource = hasDefinitiveNonStarSource(sourceElements, targetIdentifierName);
12497                        if (hasDefinitiveExplicitSource) {
12498                                if (!explicitStarTargetColumns.containsKey(targetColumnId)) {
12499                                        explicitStarTargetColumns.put(targetColumnId, new HashSet<String>());
12500                                }
12501                                explicitStarTargetColumns.get(targetColumnId).add(targetIdentifierName);
12502                        }
12503                }
12504
12505                // Column names that have explicit (non-star) sources for this target, across all relationships
12506                Set<String> targetExplicitNames = isUnionTarget ? null : explicitStarTargetColumns.get(targetColumnId);
12507
12508                for (RelationshipElement<?> sourceItem: sourceElements) {
12509                        Object sourceElement = sourceItem.getElement();
12510                        if (sourceElement instanceof ResultColumn) {
12511                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
12512                                if (sourceColumn.hasStarLinkColumn()) {
12513                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
12514                                        int linkColumnNameSize = linkColumnNames.size();
12515                                        for (int k = 0; k < linkColumnNameSize; k++) {
12516                                                String sourceName = linkColumnNames.get(k);
12517                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
12518                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
12519                                                                continue;
12520                                                }
12521                                                // Skip star-expanded source when an explicit (non-star) source provides
12522                                                // the same column, either in this relationship or a prior one.
12523                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
12524                                                        continue;
12525                                                }
12526                                                sourceColumn source = new sourceColumn();                                               
12527                                                
12528                                                boolean find = false;
12529                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
12530                                                        for (int i = 0; i < sourceColumn.getResultSet().getColumns().size(); i++) {
12531                                                                ResultColumn columnModel = sourceColumn.getResultSet().getColumns().get(i);
12532                                                                if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(sourceName)) {
12533                                                                        source.setId(String.valueOf(columnModel.getId()));
12534                                                                        find = true;
12535                                                                        break;
12536                                                                }
12537                                                        }
12538                                                }
12539                                                if(!find) {
12540                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + k);
12541                                                }
12542                                                source.setColumn(sourceName);
12543                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12544                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12545                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12546                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12547                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12548                                                }
12549                                                if (sourceItem.getTransforms() != null) {
12550                                                        for (Transform transform : sourceItem.getTransforms()) {
12551                                                                source.addTransform(transform);
12552                                                        }
12553                                                }
12554                                                relationElement.addSource(source);
12555                                        }
12556                                        if(relationElement.getSources().isEmpty()
12557                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))
12558                                                && sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
12559                                                for (int i = 0; i < sourceColumn.getResultSet().getColumns().size(); i++) {
12560                                                        ResultColumn columnModel = sourceColumn.getResultSet().getColumns().get(i);
12561                                                        if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equalsIgnoreCase(targetIdentifierName)) {
12562                                                                sourceColumn source = new sourceColumn();
12563                                                                source.setId(String.valueOf(columnModel.getId()));
12564                                                                source.setColumn(columnModel.getName());
12565                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12566                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12567                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12568                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12569                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12570                                                                }
12571                                                                if (sourceItem.getTransforms() != null) {
12572                                                                        for (Transform transform : sourceItem.getTransforms()) {
12573                                                                                source.addTransform(transform);
12574                                                                        }
12575                                                                }
12576                                                                relationElement.addSource(source);
12577                                                                break;
12578                                                        }
12579                                                }
12580                                        }
12581                                        if (relationElement.getSources().isEmpty()
12582                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))) {
12583                                                TObjectName sourceStarLinkColumn = new TObjectName();
12584                                                sourceStarLinkColumn.setString(targetName);
12585                                                boolean newBinding = sourceColumn.bindStarLinkColumn(sourceStarLinkColumn);
12586                                                sourceColumn source = new sourceColumn();
12587                                                String sourceName = DlineageUtil.getColumnName(sourceStarLinkColumn);
12588                                                if (!newBinding) {
12589                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
12590                                                                        + sourceColumn.indexOfStarLinkColumn(sourceStarLinkColumn));
12591                                                } else {
12592                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
12593                                                }
12594                                                source.setColumn(sourceName);
12595                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12596                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12597                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12598                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12599                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12600                                                }
12601                                                if (sourceItem.getTransforms() != null) {
12602                                                        for (Transform transform : sourceItem.getTransforms()) {
12603                                                                source.addTransform(transform);
12604                                                        }
12605                                                }
12606                                                relationElement.addSource(source);
12607
12608                                                if (newBinding) {
12609                                                        table resultSetElement = null;
12610                                                        for (table t : dataflow.getResultsets()) {
12611                                                                if (t.getId().equals(String.valueOf(sourceColumn.getResultSet().getId()))) {
12612                                                                        resultSetElement = t;
12613                                                                        break;
12614                                                                }
12615                                                        }
12616                                                        if (resultSetElement != null) {
12617                                                                column columnElement = new column();
12618                                                                columnElement.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
12619                                                                columnElement.setName(sourceName);
12620                                                                if (sourceColumn.isFunction()) {
12621                                                                        columnElement.setIsFunction(String.valueOf(sourceColumn.isFunction()));
12622                                                                }
12623                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12624                                                                        columnElement.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12625                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12626                                                                }
12627                                                                resultSetElement.getColumns().add(columnElement);
12628                                                        }
12629                                                }
12630                                        }
12631                                } else {
12632                                        sourceColumn source = new sourceColumn();
12633                                        source.setId(String.valueOf(sourceColumn.getId()));
12634                                        source.setColumn(sourceColumn.getName());
12635                                        source.setStruct(sourceColumn.isStruct());
12636                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12637                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12638                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12639                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12640                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12641                                        }
12642                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
12643                                                if (!targetIdentifierName
12644                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))) {
12645                                                        if (!"*".equals(sourceColumn.getName())) {
12646                                                                continue;
12647                                                        }
12648                                                        else {
12649                                                                boolean flag = false;
12650                                                                for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
12651                                                                        if(targetIdentifierName
12652                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
12653                                                                                flag = true;
12654                                                                                break;
12655                                                                        }
12656                                                                }
12657                                                                if(flag) {
12658                                                                        continue;
12659                                                                }
12660                                                        }
12661                                                }
12662                                        }
12663                                        if (sourceItem.getTransforms() != null) {
12664                                                for (Transform transform : sourceItem.getTransforms()) {
12665                                                        source.addTransform(transform);
12666                                                }
12667                                        }
12668                                        relationElement.addSource(source);
12669                                }
12670                        } else if (sourceElement instanceof TableColumn) {
12671                                TableColumn sourceColumn = (TableColumn) sourceElement;
12672                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()) {
12673                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
12674                                        int linkColumnNameSize = linkColumnNames.size();
12675                                        for (int k = 0; k < linkColumnNameSize; k++) {
12676                                                String sourceName = linkColumnNames.get(k);
12677                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
12678                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
12679                                                                continue;
12680                                                }
12681
12682                                                TableColumn tableColumn = searchTableColumn(sourceColumn.getTable().getColumns(), sourceName);
12683
12684                                                sourceColumn source = new sourceColumn();
12685                                                if (tableColumn == null) {
12686                                                        source.setId(sourceColumn.getId() + "_" + k);
12687                                                } else {
12688                                                        source.setId(String.valueOf(tableColumn.getId()));
12689                                                }
12690                                                source.setColumn(sourceName);
12691                                                source.setStruct(sourceColumn.isStruct());
12692                                                if (containStarColumn(sourceElements, sourceName)) {
12693                                                        continue;
12694                                                }
12695                                                // Cross-relationship check: skip if explicit source was found in prior relationship
12696                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
12697                                                        continue;
12698                                                }
12699                                                if (sourceColumn.getTable().getColumns().size() > 1) {
12700                                                        for (int y = 0; y < sourceColumn.getTable().getColumns().size(); y++) {
12701                                                                if (sourceColumn.getTable().getColumns().get(y).getName()
12702                                                                                .equalsIgnoreCase(sourceName)) {
12703                                                                        source.setId(String.valueOf(sourceColumn.getTable().getColumns().get(y).getId()));
12704                                                                        break;
12705                                                                }
12706                                                        }
12707                                                }
12708                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12709                                                source.setParent_name(getTableName(sourceColumn.getTable()));
12710                                                if (sourceItem instanceof TableColumnRelationshipElement) {
12711                                                        source.setParent_alias(
12712                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12713                                                }
12714                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12715                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12716                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12717                                                }
12718                                                if (sourceItem.getTransforms() != null) {
12719                                                        for (Transform transform : sourceItem.getTransforms()) {
12720                                                                source.addTransform(transform);
12721                                                        }
12722                                                }
12723                                                relationElement.addSource(source);
12724                                        }
12725                                } else {
12726                                        sourceColumn source = new sourceColumn();
12727                                        source.setId(String.valueOf(sourceColumn.getId()));
12728                                        source.setColumn(sourceColumn.getName());
12729                                        source.setStruct(sourceColumn.isStruct());
12730                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12731                                        source.setParent_name(getTableName(sourceColumn.getTable()));
12732                                        if (sourceItem instanceof TableColumnRelationshipElement) {
12733                                                source.setParent_alias(((TableColumnRelationshipElement) sourceItem).getTableAlias());
12734                                        }
12735                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12736                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12737                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12738                                        }
12739                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
12740                                                if (!targetIdentifierName
12741                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))
12742                                                                && !"*".equals(sourceColumn.getName()))
12743                                                        continue;
12744                                        }
12745                                        if (sourceItem.getTransforms() != null) {
12746                                                for (Transform transform : sourceItem.getTransforms()) {
12747                                                        source.addTransform(transform);
12748                                                }
12749                                        }
12750                                        relationElement.addSource(source);
12751                                }
12752                        }
12753                }
12754
12755                if (relationElement.getTarget() != null && relationElement.getSources() != null
12756                                && !relationElement.getSources().isEmpty()) {
12757                        dataflow.getRelationships().add(relationElement);
12758                }
12759        }
12760
12761        private String getColumnName(TObjectName column) {
12762                if (column == null) {
12763                        return null;
12764                }
12765                String name = column.getColumnNameOnly();
12766                if (name == null || "".equals(name.trim())) {
12767                        return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim());
12768                } else
12769                        return DlineageUtil.getIdentifierNormalColumnName(name.trim());
12770        }
12771
12772        /**
12773         * For BigQuery/Redshift struct field access, returns the full struct path
12774         * (e.g., "customer.name" from ColumnSource with exposedName="customer", fieldPath=["name"]).
12775         * Checks StructFieldHint first (for 3+ part no-alias), then ColumnSource (for 2-part/alias).
12776         * Returns null if this is not a struct field access.
12777         */
12778        private String getStructFieldFullName(TObjectName column) {
12779                if (column == null) return null;
12780                if (getOption().getVendor() != EDbVendor.dbvbigquery
12781                        && getOption().getVendor() != EDbVendor.dbvredshift) return null;
12782                // Priority 1: StructFieldHint (side-channel, for 3+ part no-alias deep struct access)
12783                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
12784                if (hint != null && hint.getFieldPath() != null && !hint.getFieldPath().isEmpty()) {
12785                        return hint.toFullReference();
12786                }
12787                // Priority 2: ColumnSource (main resolution, for 2-part and alias cases)
12788                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
12789                if (source != null && source.isStructFieldAccess() && source.hasFieldPath()) {
12790                        return source.getFieldPath().toFullReference(source.getExposedName());
12791                }
12792                return null;
12793        }
12794
12795        /**
12796         * Get the base column name for a struct field access column.
12797         * Checks StructFieldHint first (3+ part no-alias), then ColumnSource (2-part/alias).
12798         * Returns null if not a struct field access.
12799         */
12800        private String getStructFieldBaseName(TObjectName column) {
12801                if (column == null) return null;
12802                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
12803                if (hint != null && hint.getBaseColumn() != null) {
12804                        return hint.getBaseColumn();
12805                }
12806                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
12807                if (source != null && source.isStructFieldAccess()) {
12808                        return source.getExposedName();
12809                }
12810                return null;
12811        }
12812
12813        private String getColumnName(String column) {
12814                if (column == null) {
12815                        return null;
12816                }
12817                String name = column.substring(column.lastIndexOf(".") + 1);
12818                if (name == null || "".equals(name.trim())) {
12819                        return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim());
12820                } else
12821                        return DlineageUtil.getIdentifierNormalColumnName(name.trim());
12822        }
12823        
12824        private String getColumnNameOnly(String column) {
12825                if (column == null) {
12826                        return null;
12827                }
12828                return DlineageUtil.getColumnNameOnly(column);
12829        }
12830
12831        private void appendRecordSetRelation(dataflow dataflow, Relationship[] relations) {
12832                for (int i = 0; i < relations.length; i++) {
12833                        AbstractRelationship relation = (AbstractRelationship) relations[i];
12834                        relationship relationElement = new relationship();
12835                        relationElement.setType(relation.getRelationshipType().name());
12836                        if (relation.getFunction() != null) {
12837                                relationElement.setFunction(relation.getFunction());
12838                        }
12839                        if (relation.getEffectType() != null) {
12840                                relationElement.setEffectType(relation.getEffectType().name());
12841                        }
12842                        relationElement.setSqlHash(relation.getSqlHash());
12843                        relationElement.setSqlComment(relation.getSqlComment());
12844
12845                        if (relation.getProcedureId() != null) {
12846                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12847                        }
12848                        relationElement.setId(String.valueOf(relation.getId()));
12849
12850                        if (relation instanceof RecordSetRelationship) {
12851                                RecordSetRelationship recordCountRelation = (RecordSetRelationship) relation;
12852
12853                                Object targetElement = recordCountRelation.getTarget().getElement();
12854                                if (targetElement instanceof ResultColumn) {
12855                                        ResultColumn targetColumn = (ResultColumn) targetElement;
12856                                        targetColumn target = new targetColumn();
12857                                        target.setId(String.valueOf(targetColumn.getId()));
12858                                        target.setColumn(targetColumn.getName());
12859                                        target.setStruct(targetColumn.isStruct());
12860                                        target.setFunction(recordCountRelation.getAggregateFunction());
12861                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
12862                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
12863                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12864                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12865                                                                + convertCoordinate(targetColumn.getEndPosition()));
12866                                        }
12867                                        relationElement.setTarget(target);
12868                                } else if (targetElement instanceof TableColumn) {
12869                                        TableColumn targetColumn = (TableColumn) targetElement;
12870                                        targetColumn target = new targetColumn();
12871                                        target.setId(String.valueOf(targetColumn.getId()));
12872                                        target.setColumn(targetColumn.getName());
12873                                        target.setStruct(targetColumn.isStruct());
12874                                        target.setFunction(recordCountRelation.getAggregateFunction());
12875                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
12876                                        target.setParent_name(getTableName(targetColumn.getTable()));
12877                                        if (recordCountRelation.getTarget() instanceof TableColumnRelationshipElement) {
12878                                                target.setParent_alias(
12879                                                                ((TableColumnRelationshipElement) recordCountRelation.getTarget()).getTableAlias());
12880                                        }
12881                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12882                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12883                                                                + convertCoordinate(targetColumn.getEndPosition()));
12884                                        }
12885                                        relationElement.setTarget(target);
12886                                } else if (targetElement instanceof ResultSetRelationRows) {
12887                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
12888                                        targetColumn target = new targetColumn();
12889                                        target.setId(String.valueOf(targetColumn.getId()));
12890                                        target.setColumn(targetColumn.getName());
12891                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
12892                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
12893                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12894                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12895                                                                + convertCoordinate(targetColumn.getEndPosition()));
12896                                        }
12897                                        target.setSource("system");
12898                                        relationElement.setTarget(target);
12899                                } else {
12900                                        continue;
12901                                }
12902
12903                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)recordCountRelation.getSources();
12904                                if (sourceElements.size() == 0) {
12905                                        continue;
12906                                }
12907
12908                                boolean append = false;
12909                                for (RelationshipElement<?> sourceItem: sourceElements) {
12910                                        Object sourceElement = sourceItem.getElement();
12911                                        if (sourceElement instanceof Table) {
12912                                                Table table = (Table) sourceElement;
12913                                                sourceColumn source = new sourceColumn();
12914                                                source.setSource_id(String.valueOf(table.getId()));
12915                                                source.setSource_name(getTableName(table));
12916                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
12917                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
12918                                                                        + convertCoordinate(table.getEndPosition()));
12919                                                }
12920                                                append = true;
12921                                                relationElement.addSource(source);
12922                                        } else if (sourceElement instanceof QueryTable) {
12923                                                QueryTable table = (QueryTable) sourceElement;
12924                                                sourceColumn source = new sourceColumn();
12925                                                source.setSource_id(String.valueOf(table.getId()));
12926                                                source.setSource_name(getResultSetName(table));
12927                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
12928                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
12929                                                                        + convertCoordinate(table.getEndPosition()));
12930                                                }
12931                                                append = true;
12932                                                relationElement.addSource(source);
12933                                        } else if (sourceElement instanceof TableRelationRows) {
12934                                                TableRelationRows relationRows = (TableRelationRows) sourceElement;
12935                                                sourceColumn source = new sourceColumn();
12936                                                source.setId(String.valueOf(relationRows.getId()));
12937                                                source.setColumn(relationRows.getName());
12938                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
12939                                                source.setParent_name(getTableName(relationRows.getHolder()));
12940                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
12941                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
12942                                                                        + convertCoordinate(relationRows.getEndPosition()));
12943                                                }
12944                                                source.setSource("system");
12945                                                append = true;
12946                                                relationElement.addSource(source);
12947                                        } else if (sourceElement instanceof ResultSetRelationRows) {
12948                                                ResultSetRelationRows relationRows = (ResultSetRelationRows) sourceElement;
12949                                                sourceColumn source = new sourceColumn();
12950                                                source.setId(String.valueOf(relationRows.getId()));
12951                                                source.setColumn(relationRows.getName());
12952                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
12953                                                source.setParent_name(getResultSetName(relationRows.getHolder()));
12954                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
12955                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
12956                                                                        + convertCoordinate(relationRows.getEndPosition()));
12957                                                }
12958                                                source.setSource("system");
12959                                                append = true;
12960                                                relationElement.addSource(source);
12961                                        } else if (sourceElement instanceof TableColumn) {
12962                                                TableColumn sourceColumn = (TableColumn) sourceElement;
12963                                                sourceColumn source = new sourceColumn();
12964                                                source.setId(String.valueOf(sourceColumn.getId()));
12965                                                source.setColumn(sourceColumn.getName());
12966                                                source.setStruct(sourceColumn.isStruct());
12967                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12968                                                source.setParent_name(getTableName(sourceColumn.getTable()));
12969                                                if (sourceItem instanceof TableColumnRelationshipElement) {
12970                                                        source.setParent_alias(
12971                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12972                                                }
12973                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12974                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12975                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12976                                                }
12977                                                append = true;
12978                                                relationElement.addSource(source);
12979                                        }
12980                                        if (sourceElement instanceof ResultColumn) {
12981                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
12982                                                sourceColumn source = new sourceColumn();
12983                                                source.setId(String.valueOf(sourceColumn.getId()));
12984                                                source.setColumn(sourceColumn.getName());
12985                                                source.setStruct(sourceColumn.isStruct());
12986                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12987                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12988                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12989                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12990                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12991                                                }
12992                                                append = true;
12993                                                relationElement.addSource(source);
12994                                        }
12995                                }
12996
12997                                if (append)
12998                                        dataflow.getRelationships().add(relationElement);
12999                        }
13000                }
13001        }
13002
13003        private void appendCallRelation(dataflow dataflow, Relationship[] relations) {
13004                for (int i = 0; i < relations.length; i++) {
13005                        AbstractRelationship relation = (AbstractRelationship) relations[i];
13006                        relationship relationElement = new relationship();
13007                        relationElement.setType(relation.getRelationshipType().name());
13008                        if (relation.getFunction() != null) {
13009                                relationElement.setFunction(relation.getFunction());
13010                        }
13011                        if (relation.getEffectType() != null) {
13012                                relationElement.setEffectType(relation.getEffectType().name());
13013                        }
13014                        relationElement.setSqlHash(relation.getSqlHash());
13015                        relationElement.setSqlComment(relation.getSqlComment());
13016                        
13017                        if (relation.getProcedureId() != null) {
13018                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
13019                        }
13020                        relationElement.setId(String.valueOf(relation.getId()));
13021
13022                        if (relation instanceof CallRelationship) {
13023                                CallRelationship callRelation = (CallRelationship) relation;
13024                                
13025                                if (callRelation.getCallObject() != null) {
13026                                        relationElement.setCallStmt(callRelation.getCallObject().toString());
13027                                        if (callRelation.getStartPosition() != null && callRelation.getEndPosition() != null) {
13028                                                relationElement.setCallCoordinate(convertCoordinate(callRelation.getStartPosition()) + ","
13029                                                                + convertCoordinate(callRelation.getEndPosition()));
13030                                        }
13031                                }
13032                                
13033                                if (Boolean.TRUE.equals(callRelation.getBuiltIn())) {
13034                                        relationElement.setBuiltIn(true);
13035                                }
13036                                Object targetElement = callRelation.getTarget().getElement();
13037                                if (targetElement instanceof Procedure) {
13038                                        Procedure procedure = (Procedure) targetElement;
13039                                        targetColumn target = new targetColumn();
13040                                        target.setId(String.valueOf(procedure.getId()));
13041                                        target.setName(getProcedureName(procedure));
13042                                        if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
13043                                                target.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
13044                                                                + convertCoordinate(procedure.getEndPosition()));
13045                                        }
13046                                        String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
13047                                        if (clazz.indexOf("function") != -1) {
13048                                                target.setType("function");
13049                                        } else if (clazz.indexOf("trigger") != -1) {
13050                                                target.setType("trigger");
13051                                        } else if (clazz.indexOf("macro") != -1) {
13052                                                target.setType("macro");
13053                                        } else {
13054                                                target.setType("procedure");
13055                                        }
13056                                        relationElement.setCaller(target);
13057                                } else {
13058                                        continue;
13059                                }
13060
13061                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)callRelation.getSources();
13062                                if (sourceElements.size() == 0) {
13063                                        continue;
13064                                }
13065
13066                                boolean append = false;
13067                                for (RelationshipElement<?> sourceItem: sourceElements) {
13068                                        Object sourceElement = sourceItem.getElement();
13069                                        if (sourceElement instanceof Procedure) {
13070                                                Procedure procedure = (Procedure) sourceElement;
13071                                                sourceColumn source = new sourceColumn();
13072                                                source.setId(String.valueOf(procedure.getId()));
13073                                                source.setName(getProcedureName(procedure));
13074                                                if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
13075                                                        source.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
13076                                                                        + convertCoordinate(procedure.getEndPosition()));
13077                                                }
13078                                                String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
13079                                                if (clazz.indexOf("function") != -1) {
13080                                                        source.setType("function");
13081                                                } else if (clazz.indexOf("trigger") != -1) {
13082                                                        source.setType("trigger");
13083                                                } else if (clazz.indexOf("macro") != -1) {
13084                                                        source.setType("macro");
13085                                                } else {
13086                                                        source.setType("procedure");
13087                                                }
13088                                                append = true;
13089                                                relationElement.getCallees().add(source);
13090                                        } else if (sourceElement instanceof Function) {
13091                                                Function function = (Function) sourceElement;
13092                                                sourceColumn source = new sourceColumn();
13093                                                source.setId(String.valueOf(function.getId()));
13094                                                source.setName(getFunctionName(function.getFunctionObject()));
13095                                                if (function.getStartPosition() != null && function.getEndPosition() != null) {
13096                                                        source.setCoordinate(convertCoordinate(function.getStartPosition()) + ","
13097                                                                        + convertCoordinate(function.getEndPosition()));
13098                                                }
13099                                                source.setType("function");
13100                                                append = true;
13101                                                relationElement.getCallees().add(source);
13102                                        }
13103                                }
13104
13105                                if (append)
13106                                        dataflow.getRelationships().add(relationElement);
13107                        }
13108                }
13109        }
13110
13111        private void appendResultSets(dataflow dataflow) {
13112                Set<ResultSet> resultSets = modelManager.getResultSets(); 
13113                for (ResultSet resultSet: resultSets) {
13114                        appendResultSet(dataflow, resultSet);
13115                }
13116        }
13117
13118        private void appendResultSet(dataflow dataflow, ResultSet resultSetModel) {
13119                if (!appendResultSets.contains(resultSetModel)) {
13120                        appendResultSets.add(resultSetModel);
13121                } else {
13122                        return;
13123                }
13124
13125                table resultSetElement = new table();
13126                resultSetElement.setId(String.valueOf(resultSetModel.getId()));
13127                resultSetElement.setServer(resultSetModel.getServer());
13128                if (!SQLUtil.isEmpty(resultSetModel.getDatabase())) {
13129                        resultSetElement.setDatabase(resultSetModel.getDatabase());
13130                }
13131                if (!SQLUtil.isEmpty(resultSetModel.getSchema())) {
13132                        resultSetElement.setSchema(resultSetModel.getSchema());
13133                }
13134                resultSetElement.setName(getResultSetName(resultSetModel));
13135                resultSetElement.setType(getResultSetType(resultSetModel));
13136                // if ((ignoreRecordSet || simpleOutput) && resultSetModel.isTarget()) {
13137                resultSetElement.setIsTarget(String.valueOf(resultSetModel.isTarget()));
13138                // }
13139                resultSetElement.setIsDetermined(String.valueOf(resultSetModel.isDetermined()));
13140                if (resultSetModel.getStartPosition() != null && resultSetModel.getEndPosition() != null) {
13141                        resultSetElement.setCoordinate(convertCoordinate(resultSetModel.getStartPosition()) + ","
13142                                        + convertCoordinate(resultSetModel.getEndPosition()));
13143                }
13144                dataflow.getResultsets().add(resultSetElement);
13145
13146                List<ResultColumn> columns = resultSetModel.getColumns();
13147
13148                Map<String, Integer> columnCounts = new HashMap<String, Integer>();
13149                for (ResultColumn column : columns) {
13150                        String columnName = DlineageUtil.getIdentifierNormalColumnName(column.getName());
13151                        if (!columnCounts.containsKey(columnName)) {
13152                                columnCounts.put(columnName, 0);
13153                        }
13154                        columnCounts.put(columnName, columnCounts.get(columnName) + 1);
13155                        // if (column.hasStarLinkColumn()) {
13156                        // List<String> starLinkColumns = column.getStarLinkColumnNames();
13157                        // for (int k = 0; k < starLinkColumns.size(); k++) {
13158                        // columnName = starLinkColumns.get(k);
13159                        // if (!columnCounts.containsKey(columnName)) {
13160                        // columnCounts.put(columnName, 0);
13161                        // }
13162                        // columnCounts.put(columnName, columnCounts.get(columnName) + 1);
13163                        // }
13164                        // }
13165                }
13166
13167                for (int j = 0; j < columns.size(); j++) {
13168                        ResultColumn columnModel = columns.get(j);
13169                        if (columnModel.hasStarLinkColumn()) {
13170                                // List<String> starLinkColumns =
13171                                // columnModel.getStarLinkColumnNames();
13172                                // for (int k = 0; k < starLinkColumns.size(); k++) {
13173                                // column columnElement = new column();
13174                                // columnElement.setId( String.valueOf(columnModel.getId()) +
13175                                // "_" + k);
13176                                // String columnName = starLinkColumns.get(k);
13177                                // columnElement.setName(columnName);
13178                                // if(columnModel.isFunction()){
13179                                // columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
13180                                // }
13181                                // if (columnModel.getStartPosition() != null &&
13182                                // columnModel.getEndPosition() != null) {
13183                                // columnElement.setCoordinate(
13184                                // columnModel.getStartPosition() + "," +
13185                                // columnModel.getEndPosition());
13186                                // }
13187                                // String identifier = columnName;
13188                                // if(columnCounts.containsKey(identifier) &&
13189                                // columnCounts.get(identifier)>1){
13190                                // TObjectName column =
13191                                // columnModel.getStarLinkColumns().get(columnName).iterator().next();
13192                                // if(!SQLUtil.isEmpty(getQualifiedTable(column))){
13193                                // columnElement.setQualifiedTable(getQualifiedTable(column));
13194                                // }
13195                                // }
13196                                // resultSetElement.getColumns().add(columnElement);
13197                                // }
13198                                if (columnModel.isShowStar()) {
13199                                        column columnElement = new column();
13200                                        columnElement.setId(String.valueOf(columnModel.getId()));
13201                                        columnElement.setName(columnModel.getName());
13202                                        if (columnModel.isFunction()) {
13203                                                columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
13204                                        }
13205                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13206                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13207                                                                + convertCoordinate(columnModel.getEndPosition()));
13208                                        }
13209
13210                                        String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
13211                                        if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
13212                                                String qualifiedTable = getQualifiedTable(columnModel);
13213                                                if (!SQLUtil.isEmpty(qualifiedTable)) {
13214                                                        columnElement.setQualifiedTable(qualifiedTable);
13215                                                }
13216                                        }
13217                                        resultSetElement.getColumns().add(columnElement);
13218                                }
13219                        } else {
13220                                column columnElement = new column();
13221                                columnElement.setId(String.valueOf(columnModel.getId()));
13222                                columnElement.setName(columnModel.getName());
13223                                if (columnModel.isFunction()) {
13224                                        columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
13225                                }
13226                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13227                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13228                                                        + convertCoordinate(columnModel.getEndPosition()));
13229                                }
13230
13231                                String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
13232                                if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
13233                                        String qualifiedTable = getQualifiedTable(columnModel);
13234                                        if (!SQLUtil.isEmpty(qualifiedTable)) {
13235                                                columnElement.setQualifiedTable(qualifiedTable);
13236                                        }
13237                                }
13238                                resultSetElement.getColumns().add(columnElement);
13239                        }
13240                }
13241
13242                ResultSetRelationRows relationRows = resultSetModel.getRelationRows();
13243                if (relationRows.hasRelation()) {
13244                        column relationRowsElement = new column();
13245                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13246                        relationRowsElement.setName(relationRows.getName());
13247                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13248                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13249                                                + convertCoordinate(relationRows.getEndPosition()));
13250                        }
13251                        relationRowsElement.setSource("system");
13252                        resultSetElement.getColumns().add(relationRowsElement);
13253                }
13254        }
13255
13256        private String getQualifiedTable(ResultColumn columnModel) {
13257                if (columnModel.getColumnObject() instanceof TObjectName) {
13258                        return getQualifiedTable((TObjectName) columnModel.getColumnObject());
13259                }
13260                if (columnModel.getColumnObject() instanceof TResultColumn) {
13261                        TObjectName field = ((TResultColumn) columnModel.getColumnObject()).getFieldAttr();
13262                        if (field != null) {
13263                                return getQualifiedTable(field);
13264                        }
13265                }
13266                return null;
13267        }
13268
13269        private String getQualifiedTable(TObjectName column) {
13270                if (column == null)
13271                        return null;
13272                String[] splits = column.toString().split("\\.");
13273                if (splits.length > 1) {
13274                        return splits[splits.length - 2];
13275                }
13276                return null;
13277        }
13278
13279        /**
13280         * Get the qualified prefix (schema.table) from a column name for 3-part names.
13281         * For example, for "sch.pk_constv2.c_cdsl", returns "sch.pk_constv2".
13282         * Returns null if the column doesn't have both schema and table parts.
13283         */
13284        private String getQualifiedPrefixFromColumn(TObjectName column) {
13285                if (column == null) return null;
13286
13287                // Check if both schema and table tokens are present (3-part name)
13288                String schemaStr = column.getSchemaString();
13289                String tableStr = column.getTableString();
13290
13291                if (schemaStr != null && !schemaStr.isEmpty() &&
13292                        tableStr != null && !tableStr.isEmpty()) {
13293                        return schemaStr + "." + tableStr;
13294                }
13295
13296                // Fallback: parse from toString() for complex cases
13297                String[] splits = column.toString().split("\\.");
13298                if (splits.length >= 3) {
13299                        // Return all parts except the last one (column name)
13300                        StringBuilder prefix = new StringBuilder();
13301                        for (int i = 0; i < splits.length - 1; i++) {
13302                                if (i > 0) prefix.append(".");
13303                                prefix.append(splits[i]);
13304                        }
13305                        return prefix.toString();
13306                }
13307
13308                return null;
13309        }
13310
13311        private String getResultSetType(ResultSet resultSetModel) {
13312                if (resultSetModel instanceof QueryTable) {
13313                        QueryTable table = (QueryTable) resultSetModel;
13314                        if (table.getTableObject().getCTE() != null) {
13315                                return "with_cte";
13316                        }
13317                }
13318
13319                if (resultSetModel instanceof SelectSetResultSet) {
13320                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
13321                        return "select_" + type.name();
13322                }
13323
13324                if (resultSetModel instanceof SelectResultSet) {
13325                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
13326                                return "insert-select";
13327                        }
13328                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
13329                                return "update-select";
13330                        }
13331                }
13332
13333                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
13334                        return "merge-update";
13335                }
13336
13337                if (resultSetModel.getGspObject() instanceof TOutputClause) {
13338                        return ResultSetType.output.name();
13339                }
13340
13341                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
13342                        return "merge-insert";
13343                }
13344
13345                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
13346                        return "update-set";
13347                }
13348
13349                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
13350                        return ResultSetType.array.name();
13351                }
13352
13353                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
13354                        return ResultSetType.struct.name();
13355                }
13356
13357                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
13358                        return ResultSetType.function.name();
13359                }
13360
13361                if (resultSetModel.getGspObject() instanceof TAliasClause) {
13362                        return ResultSetType.alias.name();
13363                }
13364
13365                if (resultSetModel.getGspObject() instanceof TCursorDeclStmt) {
13366                        return ResultSetType.cursor.name();
13367                }
13368
13369                if (resultSetModel instanceof PivotedTable) {
13370                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
13371                                return ResultSetType.unpivot_table.name();
13372                        }
13373                        return ResultSetType.pivot_table.name();
13374                }
13375
13376                return "select_list";
13377        }
13378
13379        private String getTableName(Table tableModel) {
13380                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
13381                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
13382                }
13383
13384                String tableName;
13385                if (tableModel.getFullName() != null && tableModel.getFullName().trim().length() > 0) {
13386                        return tableModel.getFullName();
13387                }
13388                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
13389                        tableName = getResultSetWithId("RESULT_OF_" + tableModel.getAlias());
13390
13391                } else {
13392                        tableName = getResultSetDisplayId("RS");
13393                }
13394                modelManager.DISPLAY_NAME.put(tableModel.getId(), tableName);
13395                return tableName;
13396        }
13397
13398        private String getProcedureName(Procedure procedureModel) {
13399                if (modelManager.DISPLAY_NAME.containsKey(procedureModel.getId())) {
13400                        return modelManager.DISPLAY_NAME.get(procedureModel.getId());
13401                }
13402
13403                String procedureName = procedureModel.getFullName();
13404
13405                modelManager.DISPLAY_NAME.put(procedureModel.getId(), procedureName);
13406                return procedureName;
13407        }
13408
13409        private String getProcessName(Process processModel) {
13410                if (modelManager.DISPLAY_NAME.containsKey(processModel.getId())) {
13411                        return modelManager.DISPLAY_NAME.get(processModel.getId());
13412                } else {
13413                        if (processModel.getCustomType() != null) {
13414                                String name = processModel.getCustomType();
13415                                modelManager.DISPLAY_NAME.put(processModel.getId(), name);
13416                                return name;
13417                        }
13418                        String name = processModel.getType();
13419                        String procedureName = getProcedureParentName(processModel.getGspObject());
13420                        if (procedureName != null) {
13421                                name = getResultSetDisplayId(procedureName + " " + name);
13422                        } else {
13423                                name = getResultSetDisplayId("Query " + name);
13424                        }
13425                        modelManager.DISPLAY_NAME.put(processModel.getId(), name);
13426                        return name;
13427                }
13428        }
13429
13430        private String getDisplayIdByType(String type) {
13431                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13432                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
13433                        return type + "-" + (option.getStartId() + 1);
13434                } else {
13435                        long id = modelManager.DISPLAY_ID.get(type);
13436                        modelManager.DISPLAY_ID.put(type, id + 1);
13437                        return type + "-" + (id + 1);
13438                }
13439        }
13440        
13441        private String getDisplayIdByTypeFromZero(String type) {
13442                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13443                        modelManager.DISPLAY_ID.put(type, option.getStartId());
13444                        if(option.getStartId() == 0) {
13445                                return type;
13446                        }
13447                        return type + "-" + (option.getStartId() + 1);
13448                } else {
13449                        long id = modelManager.DISPLAY_ID.get(type);
13450                        modelManager.DISPLAY_ID.put(type, id + 1);
13451                        return type + "-" + (id + 1);
13452                }
13453        }
13454
13455        private String getConstantName(Table tableModel) {
13456                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
13457                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
13458                } else {
13459                        String name = getDisplayIdByType("SQL_CONSTANTS");
13460                        modelManager.DISPLAY_NAME.put(tableModel.getId(), name);
13461                        return name;
13462                }
13463        }
13464        
13465        private String getTempTableName(TTable table) {
13466                if (modelManager.DISPLAY_NAME.containsKey((long)System.identityHashCode(table))) {
13467                        return modelManager.DISPLAY_NAME.get((long)System.identityHashCode(table));
13468                } else {
13469                        String name = getDisplayIdByTypeFromZero(table.getName());
13470                        modelManager.DISPLAY_NAME.put((long)System.identityHashCode(table), name);
13471                        return name;
13472                }
13473        }
13474
13475        private String getResultSetName(ResultSet resultSetModel) {
13476
13477                if (modelManager.DISPLAY_NAME.containsKey(resultSetModel.getId())) {
13478                        return modelManager.DISPLAY_NAME.get(resultSetModel.getId());
13479                }
13480                
13481                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
13482                        String name = getResultSetDisplayId("ARRAY");
13483                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13484                        if(option.containsResultSetType(ResultSetType.array)) {
13485                                resultSetModel.setTarget(true);
13486                        }
13487                        return name;
13488                }
13489                
13490                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
13491                        String name = getResultSetDisplayId("STRUCT");
13492                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13493                        if(option.containsResultSetType(ResultSetType.struct)) {
13494                                resultSetModel.setTarget(true);
13495                        }
13496                        return name;
13497                }
13498
13499                if (resultSetModel instanceof QueryTable) {
13500                        QueryTable table = (QueryTable) resultSetModel;
13501                        if (table.getAlias() != null && table.getAlias().trim().length() > 0) {
13502                                String name = getResultSetWithId("RESULT_OF_" + table.getAlias().trim());
13503                                if (table.getTableObject().getCTE() != null) {
13504                                        name = getResultSetWithId("RESULT_OF_" + table.getTableObject().getCTE().getTableName().toString()
13505                                                        + "_" + table.getAlias().trim());
13506                                }
13507                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13508                                if(option.containsResultSetType(ResultSetType.result_of)) {
13509                                        resultSetModel.setTarget(true);
13510                                }
13511                                return name;
13512                        } else if (table.getTableObject().getCTE() != null) {
13513                                String name = getResultSetWithId("CTE-" + table.getTableObject().getCTE().getTableName().toString());
13514                                modelManager.DISPLAY_NAME.put(table.getId(), name);
13515                                if(option.containsResultSetType(ResultSetType.cte)) {
13516                                        resultSetModel.setTarget(true);
13517                                }
13518                                return name;
13519                        }
13520                }
13521
13522                if (resultSetModel instanceof SelectResultSet) {
13523                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
13524                                String name = getResultSetDisplayId("INSERT-SELECT");
13525                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13526                                if(option.containsResultSetType(ResultSetType.insert_select)) {
13527                                        resultSetModel.setTarget(true);
13528                                }
13529                                return name;
13530                        }
13531
13532                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
13533                                String name = getResultSetDisplayId("UPDATE-SELECT");
13534                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13535                                if(option.containsResultSetType(ResultSetType.update_select)) {
13536                                        resultSetModel.setTarget(true);
13537                                }
13538                                return name;
13539                        }
13540                }
13541
13542                if (resultSetModel instanceof SelectSetResultSet) {
13543                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
13544                        String name = getResultSetDisplayId("RESULT_OF_" + type.name().toUpperCase());
13545                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13546                        if(option.containsResultSetType(ResultSetType.result_of)) {
13547                                resultSetModel.setTarget(true);
13548                        }
13549                        return name;
13550                }
13551
13552                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
13553                        String name = getResultSetDisplayId("MERGE-UPDATE");
13554                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13555                        if(option.containsResultSetType(ResultSetType.merge_update)) {
13556                                resultSetModel.setTarget(true);
13557                        }
13558                        return name;
13559                }
13560
13561                if (resultSetModel.getGspObject() instanceof TOutputClause) {
13562                        String name = getResultSetDisplayId("OUTPUT");
13563                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13564                        if(option.containsResultSetType(ResultSetType.output)) {
13565                                resultSetModel.setTarget(true);
13566                        }
13567                        return name;
13568                }
13569
13570                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
13571                        String name = getResultSetDisplayId("MERGE-INSERT");
13572                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13573                        if(option.containsResultSetType(ResultSetType.merge_insert)) {
13574                                resultSetModel.setTarget(true);
13575                        }
13576                        return name;
13577                }
13578
13579                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
13580                        String name = getResultSetDisplayId("UPDATE-SET");
13581                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13582                        if(option.containsResultSetType(ResultSetType.update_set)) {
13583                                resultSetModel.setTarget(true);
13584                        }
13585                        return name;
13586                }
13587
13588                if (resultSetModel.getGspObject() instanceof TCaseExpression) {
13589                        String name = ((Function) resultSetModel).getFunctionName();
13590                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13591                        if (option.containsResultSetType(ResultSetType.case_when) || option.containsResultSetType(ResultSetType.function)) {
13592                                resultSetModel.setTarget(true);
13593                        }
13594                        return name;
13595                }
13596                
13597                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
13598                        String name = ((Function) resultSetModel).getFunctionName();
13599                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13600                        if(option.containsResultSetType(ResultSetType.function)) {
13601                                resultSetModel.setTarget(true);
13602                        }
13603                        return name;
13604                }
13605
13606                if (resultSetModel instanceof PivotedTable) {
13607                        String name = getResultSetDisplayId("PIVOT-TABLE");
13608                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
13609                                name = getResultSetDisplayId("UNPIVOT-TABLE");
13610                                if(option.containsResultSetType(ResultSetType.unpivot_table)) {
13611                                        resultSetModel.setTarget(true);
13612                                }
13613                        }
13614                        else {
13615                                if(option.containsResultSetType(ResultSetType.pivot_table)) {
13616                                        resultSetModel.setTarget(true);
13617                                }
13618                        }
13619                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13620                        return name;
13621                }
13622
13623                if (resultSetModel instanceof Alias) {
13624                        String name = getResultSetDisplayId("ALIAS");
13625                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13626                        if(option.containsResultSetType(ResultSetType.alias)) {
13627                                resultSetModel.setTarget(true);
13628                        }
13629                        return name;
13630                }
13631
13632                String name = getResultSetDisplayId("RS");
13633                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13634                if(option.containsResultSetType(ResultSetType.select_list)) {
13635                        resultSetModel.setTarget(true);
13636                }
13637                return name;
13638        }
13639
13640        private String getResultSetWithId(String type) {
13641                type = DlineageUtil.getIdentifierNormalTableName(type);
13642                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13643                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
13644                        return type + "-" + (option.getStartId() + 1);
13645                } else {
13646                        long id = modelManager.DISPLAY_ID.get(type);
13647                        modelManager.DISPLAY_ID.put(type, id + 1);
13648                        return type + "-" + (id + 1);
13649                }
13650        }
13651
13652        private String getResultSetDisplayId(String type) {
13653                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13654                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
13655                        return type + "-" + (option.getStartId() + 1);
13656                } else {
13657                        long id = modelManager.DISPLAY_ID.get(type);
13658                        modelManager.DISPLAY_ID.put(type, id + 1);
13659                        return type + "-" + (id + 1);
13660                }
13661        }
13662
13663        private void appendViews(dataflow dataflow) {
13664                List<TCustomSqlStatement> views = modelManager.getViews();
13665                for (int i = 0; i < views.size(); i++) {
13666                        Table viewModel = (Table) modelManager.getViewModel(views.get(i));
13667                        if (!tableIds.contains(viewModel.getId())) {
13668                                appendViewModel(dataflow, viewModel);
13669                                tableIds.add(viewModel.getId());
13670                        }
13671                }
13672
13673                List<TTable> tables = modelManager.getBaseTables();
13674                for (int i = 0; i < tables.size(); i++) {
13675                        Object model = modelManager.getModel(tables.get(i));
13676                        if (model instanceof Table) {
13677                                Table tableModel = (Table) model;
13678                                if (tableModel.isView()) {
13679                                        if (!tableIds.contains(tableModel.getId())) {
13680                                                appendViewModel(dataflow, tableModel);
13681                                                tableIds.add(tableModel.getId());
13682                                        }
13683                                }
13684                        }
13685                }
13686
13687                List<Table> tableNames = modelManager.getTablesByName();
13688                for (int i = 0; i < tableNames.size(); i++) {
13689                        Table tableModel = tableNames.get(i);
13690                        if (tableModel.isView()) {
13691                                if (!tableIds.contains(tableModel.getId())) {
13692                                        appendViewModel(dataflow, tableModel);
13693                                        tableIds.add(tableModel.getId());
13694                                }
13695                        }
13696                }
13697        }
13698
13699        private void appendViewModel(dataflow dataflow, Table viewModel) {
13700                table viewElement = new table();
13701                viewElement.setId(String.valueOf(viewModel.getId()));
13702                if (!SQLUtil.isEmpty(viewModel.getDatabase())) {
13703                        viewElement.setDatabase(viewModel.getDatabase());
13704                }
13705                if (!SQLUtil.isEmpty(viewModel.getSchema())) {
13706                        viewElement.setSchema(viewModel.getSchema());
13707                }
13708                viewElement.setServer(viewModel.getServer());
13709                viewElement.setName(viewModel.getName());
13710                viewElement.setType("view");
13711                viewElement.setStarStmt(viewModel.getStarStmt());
13712
13713                if(viewModel.isFromDDL()){
13714                        viewElement.setFromDDL(String.valueOf(viewModel.isFromDDL()));
13715                }
13716
13717        if(option.isTraceTablePosition()){
13718            for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:viewModel.getPositions()){
13719                viewElement.setCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
13720            }
13721        }
13722        else {
13723            viewElement.setCoordinate(convertCoordinate(viewModel.getStartPosition()) + ","
13724                    + convertCoordinate(viewModel.getEndPosition()));
13725        }
13726
13727                if (viewModel.getProcesses() != null) {
13728                        List<String> processIds = new ArrayList<String>();
13729                        for (Process process : viewModel.getProcesses()) {
13730                                processIds.add(String.valueOf(process.getId()));
13731                        }
13732                        viewElement.setProcessIds(processIds);
13733                }
13734                dataflow.getViews().add(viewElement);
13735
13736                List<TableColumn> columns = viewModel.getColumns();
13737
13738                if (containStarColumn(columns)) {
13739                        for (TableColumn column : columns) {
13740                                if (column.getName().endsWith("*")) {
13741                                        for (TableColumn starElement : columns) {
13742                                                if (starElement == column) {
13743                                                        continue;
13744                                                }
13745                                                TObjectName columnObject = starElement.getColumnObject();
13746                                                column.bindStarLinkColumn(columnObject);
13747                                        }
13748                                        if (viewModel.isCreateTable()) {
13749                                                column.setShowStar(false);
13750                                        }
13751                                }
13752                        }
13753                }
13754
13755                for (int j = 0; j < columns.size(); j++) {
13756                        TableColumn columnModel = (TableColumn) columns.get(j);
13757                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn()) {
13758                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
13759                                for (int k = 0; k < starLinkColumnList.size(); k++) {
13760                                        column columnElement = new column();
13761                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
13762                                        String columnName = starLinkColumnList.get(k);
13763                                        if (containStarColumn(columns, columnName)) {
13764                                                continue;
13765                                        }
13766                                        columnElement.setName(columnName);
13767                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13768                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13769                                                                + convertCoordinate(columnModel.getEndPosition()));
13770                                        }
13771                                        viewElement.getColumns().add(columnElement);
13772                                }
13773
13774                                if (columnModel.isShowStar()) {
13775                                        column columnElement = new column();
13776                                        columnElement.setId(String.valueOf(columnModel.getId()));
13777                                        columnElement.setName(columnModel.getName());
13778                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13779                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13780                                                                + convertCoordinate(columnModel.getEndPosition()));
13781                                        }
13782                                        viewElement.getColumns().add(columnElement);
13783                                }
13784
13785                        } else {
13786                                column columnElement = new column();
13787                                columnElement.setId(String.valueOf(columnModel.getId()));
13788                                columnElement.setName(columnModel.getName());
13789                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13790                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13791                                                        + convertCoordinate(columnModel.getEndPosition()));
13792                                }
13793                                if(columnModel.isPseduo()) {
13794                                        columnElement.setSource("system");
13795                                }
13796                                viewElement.getColumns().add(columnElement);
13797                        }
13798                }
13799
13800                TableRelationRows relationRows = viewModel.getRelationRows();
13801                if (relationRows.hasRelation()) {
13802                        column relationRowsElement = new column();
13803                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13804                        relationRowsElement.setName(relationRows.getName());
13805                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13806                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13807                                                + convertCoordinate(relationRows.getEndPosition()));
13808                        }
13809                        relationRowsElement.setSource("system");
13810                        viewElement.getColumns().add(relationRowsElement);
13811                }
13812        }
13813
13814        private void appendStreamModel(dataflow dataflow, Table streamModel) {
13815                table streamElement = new table();
13816                streamElement.setId(String.valueOf(streamModel.getId()));
13817                if (!SQLUtil.isEmpty(streamModel.getDatabase())) {
13818                        streamElement.setDatabase(streamModel.getDatabase());
13819                }
13820                if (!SQLUtil.isEmpty(streamModel.getSchema())) {
13821                        streamElement.setSchema(streamModel.getSchema());
13822                }
13823                streamElement.setServer(streamModel.getServer());
13824                streamElement.setName(streamModel.getName());
13825                streamElement.setType("stream");
13826                if (streamModel.getFileType() != null) {
13827                        streamElement.setFileType(SQLUtil.trimColumnStringQuote(streamModel.getFileType()));
13828                }
13829
13830                if (streamModel.getStartPosition() != null && streamModel.getEndPosition() != null) {
13831                        streamElement.setCoordinate(convertCoordinate(streamModel.getStartPosition()) + ","
13832                                        + convertCoordinate(streamModel.getEndPosition()));
13833                }
13834
13835                if (streamModel.getProcesses() != null) {
13836                        List<String> processIds = new ArrayList<String>();
13837                        for (Process process : streamModel.getProcesses()) {
13838                                processIds.add(String.valueOf(process.getId()));
13839                        }
13840                        streamElement.setProcessIds(processIds);
13841                }
13842                dataflow.getStreams().add(streamElement);
13843
13844                List<TableColumn> columns = streamModel.getColumns();
13845
13846                for (int j = 0; j < columns.size(); j++) {
13847                        TableColumn columnModel = (TableColumn) columns.get(j);
13848                        column columnElement = new column();
13849                        columnElement.setId(String.valueOf(columnModel.getId()));
13850                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
13851                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13852                                        + convertCoordinate(columnModel.getEndPosition()));
13853                        streamElement.getColumns().add(columnElement);
13854                }
13855
13856                TableRelationRows relationRows = streamModel.getRelationRows();
13857                if (relationRows.hasRelation()) {
13858                        column relationRowsElement = new column();
13859                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13860                        relationRowsElement.setName(relationRows.getName());
13861                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13862                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13863                                                + convertCoordinate(relationRows.getEndPosition()));
13864                        }
13865                        relationRowsElement.setSource("system");
13866                        streamElement.getColumns().add(relationRowsElement);
13867                }
13868        }
13869
13870        private void appendStageModel(dataflow dataflow, Table stageModel) {
13871                table stageElement = new table();
13872                stageElement.setId(String.valueOf(stageModel.getId()));
13873                if (!SQLUtil.isEmpty(stageModel.getDatabase())) {
13874                        stageElement.setDatabase(stageModel.getDatabase());
13875                }
13876                if (!SQLUtil.isEmpty(stageModel.getSchema())) {
13877                        stageElement.setSchema(stageModel.getSchema());
13878                }
13879                stageElement.setServer(stageModel.getServer());
13880                stageElement.setName(stageModel.getName());
13881                stageElement.setType("stage");
13882                stageElement.setLocation(stageModel.getLocation());
13883                if (stageModel.getFileType() != null) {
13884                        stageElement.setFileType(SQLUtil.trimColumnStringQuote(stageModel.getFileType()));
13885                }
13886
13887                if (stageModel.getStartPosition() != null && stageModel.getEndPosition() != null) {
13888                        stageElement.setCoordinate(convertCoordinate(stageModel.getStartPosition()) + ","
13889                                        + convertCoordinate(stageModel.getEndPosition()));
13890                }
13891
13892                if (stageModel.getProcesses() != null) {
13893                        List<String> processIds = new ArrayList<String>();
13894                        for (Process process : stageModel.getProcesses()) {
13895                                processIds.add(String.valueOf(process.getId()));
13896                        }
13897                        stageElement.setProcessIds(processIds);
13898                }
13899                dataflow.getStages().add(stageElement);
13900
13901                List<TableColumn> columns = stageModel.getColumns();
13902
13903                for (int j = 0; j < columns.size(); j++) {
13904                        TableColumn columnModel = (TableColumn) columns.get(j);
13905                        column columnElement = new column();
13906                        columnElement.setId(String.valueOf(columnModel.getId()));
13907                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
13908                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13909                                        + convertCoordinate(columnModel.getEndPosition()));
13910                        stageElement.getColumns().add(columnElement);
13911                }
13912
13913                TableRelationRows relationRows = stageModel.getRelationRows();
13914                if (relationRows.hasRelation()) {
13915                        column relationRowsElement = new column();
13916                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13917                        relationRowsElement.setName(relationRows.getName());
13918                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13919                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13920                                                + convertCoordinate(relationRows.getEndPosition()));
13921                        }
13922                        relationRowsElement.setSource("system");
13923                        stageElement.getColumns().add(relationRowsElement);
13924                }
13925        }
13926
13927        private void appendSequenceModel(dataflow dataflow, Table sequenceModel) {
13928                table sequenceElement = new table();
13929                sequenceElement.setId(String.valueOf(sequenceModel.getId()));
13930                if (!SQLUtil.isEmpty(sequenceModel.getDatabase())) {
13931                        sequenceElement.setDatabase(sequenceModel.getDatabase());
13932                }
13933                if (!SQLUtil.isEmpty(sequenceModel.getSchema())) {
13934                        sequenceElement.setSchema(sequenceModel.getSchema());
13935                }
13936                sequenceElement.setServer(sequenceModel.getServer());
13937                sequenceElement.setName(sequenceModel.getName());
13938                sequenceElement.setType("sequence");
13939                sequenceElement.setLocation(sequenceModel.getLocation());
13940                if (sequenceModel.getFileType() != null) {
13941                        sequenceElement.setFileType(SQLUtil.trimColumnStringQuote(sequenceModel.getFileType()));
13942                }
13943
13944                if (sequenceModel.getStartPosition() != null && sequenceModel.getEndPosition() != null) {
13945                        sequenceElement.setCoordinate(convertCoordinate(sequenceModel.getStartPosition()) + ","
13946                                        + convertCoordinate(sequenceModel.getEndPosition()));
13947                }
13948
13949                if (sequenceModel.getProcesses() != null) {
13950                        List<String> processIds = new ArrayList<String>();
13951                        for (Process process : sequenceModel.getProcesses()) {
13952                                processIds.add(String.valueOf(process.getId()));
13953                        }
13954                        sequenceElement.setProcessIds(processIds);
13955                }
13956                dataflow.getSequences().add(sequenceElement);
13957
13958                List<TableColumn> columns = sequenceModel.getColumns();
13959
13960                for (int j = 0; j < columns.size(); j++) {
13961                        TableColumn columnModel = (TableColumn) columns.get(j);
13962                        column columnElement = new column();
13963                        columnElement.setId(String.valueOf(columnModel.getId()));
13964                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
13965                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13966                                        + convertCoordinate(columnModel.getEndPosition()));
13967                        sequenceElement.getColumns().add(columnElement);
13968                }
13969
13970                TableRelationRows relationRows = sequenceModel.getRelationRows();
13971                if (relationRows.hasRelation()) {
13972                        column relationRowsElement = new column();
13973                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13974                        relationRowsElement.setName(relationRows.getName());
13975                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13976                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13977                                                + convertCoordinate(relationRows.getEndPosition()));
13978                        }
13979                        relationRowsElement.setSource("system");
13980                        sequenceElement.getColumns().add(relationRowsElement);
13981                }
13982        }
13983
13984        private void appendDataSourceModel(dataflow dataflow, Table datasourceModel) {
13985                table datasourceElement = new table();
13986                datasourceElement.setId(String.valueOf(datasourceModel.getId()));
13987                if (!SQLUtil.isEmpty(datasourceModel.getDatabase())) {
13988                        datasourceElement.setDatabase(datasourceModel.getDatabase());
13989                }
13990                if (!SQLUtil.isEmpty(datasourceModel.getSchema())) {
13991                        datasourceElement.setSchema(datasourceModel.getSchema());
13992                }
13993                datasourceElement.setServer(datasourceModel.getServer());
13994                datasourceElement.setName(datasourceModel.getName());
13995                datasourceElement.setType("datasource");
13996                datasourceElement.setLocation(datasourceModel.getLocation());
13997                if (datasourceModel.getFileType() != null) {
13998                        datasourceElement.setFileType(SQLUtil.trimColumnStringQuote(datasourceModel.getFileType()));
13999                }
14000
14001                if (datasourceModel.getStartPosition() != null && datasourceModel.getEndPosition() != null) {
14002                        datasourceElement.setCoordinate(convertCoordinate(datasourceModel.getStartPosition()) + ","
14003                                        + convertCoordinate(datasourceModel.getEndPosition()));
14004                }
14005
14006                if (datasourceModel.getProcesses() != null) {
14007                        List<String> processIds = new ArrayList<String>();
14008                        for (Process process : datasourceModel.getProcesses()) {
14009                                processIds.add(String.valueOf(process.getId()));
14010                        }
14011                        datasourceElement.setProcessIds(processIds);
14012                }
14013                dataflow.getDatasources().add(datasourceElement);
14014
14015                List<TableColumn> columns = datasourceModel.getColumns();
14016
14017                for (int j = 0; j < columns.size(); j++) {
14018                        TableColumn columnModel = (TableColumn) columns.get(j);
14019                        column columnElement = new column();
14020                        columnElement.setId(String.valueOf(columnModel.getId()));
14021                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14022                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14023                                        + convertCoordinate(columnModel.getEndPosition()));
14024                        datasourceElement.getColumns().add(columnElement);
14025                }
14026
14027                TableRelationRows relationRows = datasourceModel.getRelationRows();
14028                if (relationRows.hasRelation()) {
14029                        column relationRowsElement = new column();
14030                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14031                        relationRowsElement.setName(relationRows.getName());
14032                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14033                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14034                                                + convertCoordinate(relationRows.getEndPosition()));
14035                        }
14036                        relationRowsElement.setSource("system");
14037                        datasourceElement.getColumns().add(relationRowsElement);
14038                }
14039        }
14040
14041        private void appendDatabaseModel(dataflow dataflow, Table databaseModel) {
14042                table databaseElement = new table();
14043                databaseElement.setId(String.valueOf(databaseModel.getId()));
14044                if (!SQLUtil.isEmpty(databaseModel.getDatabase())) {
14045                        databaseElement.setDatabase(databaseModel.getDatabase());
14046                }
14047                if (!SQLUtil.isEmpty(databaseModel.getSchema())) {
14048                        databaseElement.setSchema(databaseModel.getSchema());
14049                }
14050                databaseElement.setServer(databaseModel.getServer());
14051                databaseElement.setName(databaseModel.getName());
14052                databaseElement.setType("database");
14053                if (databaseModel.getFileType() != null) {
14054                        databaseElement.setFileType(SQLUtil.trimColumnStringQuote(databaseModel.getFileType()));
14055                }
14056
14057                if (databaseModel.getStartPosition() != null && databaseModel.getEndPosition() != null) {
14058                        databaseElement.setCoordinate(convertCoordinate(databaseModel.getStartPosition()) + ","
14059                                        + convertCoordinate(databaseModel.getEndPosition()));
14060                }
14061
14062                if (databaseModel.getProcesses() != null) {
14063                        List<String> processIds = new ArrayList<String>();
14064                        for (Process process : databaseModel.getProcesses()) {
14065                                processIds.add(String.valueOf(process.getId()));
14066                        }
14067                        databaseElement.setProcessIds(processIds);
14068                }
14069                dataflow.getDatabases().add(databaseElement);
14070
14071                List<TableColumn> columns = databaseModel.getColumns();
14072
14073                for (int j = 0; j < columns.size(); j++) {
14074                        TableColumn columnModel = (TableColumn) columns.get(j);
14075                        column columnElement = new column();
14076                        columnElement.setId(String.valueOf(columnModel.getId()));
14077                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14078                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14079                                        + convertCoordinate(columnModel.getEndPosition()));
14080                        databaseElement.getColumns().add(columnElement);
14081                }
14082
14083                TableRelationRows relationRows = databaseModel.getRelationRows();
14084                if (relationRows.hasRelation()) {
14085                        column relationRowsElement = new column();
14086                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14087                        relationRowsElement.setName(relationRows.getName());
14088                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14089                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14090                                                + convertCoordinate(relationRows.getEndPosition()));
14091                        }
14092                        relationRowsElement.setSource("system");
14093                        databaseElement.getColumns().add(relationRowsElement);
14094                }
14095        }
14096
14097        private void appendSchemaModel(dataflow dataflow, Table schemaModel) {
14098                table schemaElement = new table();
14099                schemaElement.setId(String.valueOf(schemaModel.getId()));
14100                if (!SQLUtil.isEmpty(schemaModel.getDatabase())) {
14101                        schemaElement.setDatabase(schemaModel.getDatabase());
14102                }
14103                if (!SQLUtil.isEmpty(schemaModel.getSchema())) {
14104                        schemaElement.setSchema(schemaModel.getSchema());
14105                }
14106                schemaElement.setServer(schemaModel.getServer());
14107                schemaElement.setName(schemaModel.getName());
14108                schemaElement.setType("schema");
14109                if (schemaModel.getFileType() != null) {
14110                        schemaElement.setFileType(SQLUtil.trimColumnStringQuote(schemaModel.getFileType()));
14111                }
14112
14113                if (schemaModel.getStartPosition() != null && schemaModel.getEndPosition() != null) {
14114                        schemaElement.setCoordinate(convertCoordinate(schemaModel.getStartPosition()) + ","
14115                                        + convertCoordinate(schemaModel.getEndPosition()));
14116                }
14117
14118                if (schemaModel.getProcesses() != null) {
14119                        List<String> processIds = new ArrayList<String>();
14120                        for (Process process : schemaModel.getProcesses()) {
14121                                processIds.add(String.valueOf(process.getId()));
14122                        }
14123                        schemaElement.setProcessIds(processIds);
14124                }
14125                dataflow.getSchemas().add(schemaElement);
14126
14127                List<TableColumn> columns = schemaModel.getColumns();
14128
14129                for (int j = 0; j < columns.size(); j++) {
14130                        TableColumn columnModel = (TableColumn) columns.get(j);
14131                        column columnElement = new column();
14132                        columnElement.setId(String.valueOf(columnModel.getId()));
14133                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14134                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14135                                        + convertCoordinate(columnModel.getEndPosition()));
14136                        schemaElement.getColumns().add(columnElement);
14137                }
14138
14139                TableRelationRows relationRows = schemaModel.getRelationRows();
14140                if (relationRows.hasRelation()) {
14141                        column relationRowsElement = new column();
14142                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14143                        relationRowsElement.setName(relationRows.getName());
14144                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14145                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14146                                                + convertCoordinate(relationRows.getEndPosition()));
14147                        }
14148                        relationRowsElement.setSource("system");
14149                        schemaElement.getColumns().add(relationRowsElement);
14150                }
14151        }
14152
14153        private void appendPathModel(dataflow dataflow, Table pathModel) {
14154                table pathElement = new table();
14155                pathElement.setId(String.valueOf(pathModel.getId()));
14156                if (!SQLUtil.isEmpty(pathModel.getDatabase())) {
14157                        pathElement.setDatabase(pathModel.getDatabase());
14158                }
14159                if (!SQLUtil.isEmpty(pathModel.getSchema())) {
14160                        pathElement.setSchema(pathModel.getSchema());
14161                }
14162                pathElement.setServer(pathModel.getServer());
14163                pathElement.setName(pathModel.getName());
14164                pathElement.setType("path");
14165                if (pathModel.getFileFormat() != null) {
14166                        pathElement.setFileFormat(SQLUtil.trimColumnStringQuote(pathModel.getFileFormat()));
14167                }
14168
14169                if (pathModel.getStartPosition() != null && pathModel.getEndPosition() != null) {
14170                        pathElement.setCoordinate(convertCoordinate(pathModel.getStartPosition()) + ","
14171                                        + convertCoordinate(pathModel.getEndPosition()));
14172                }
14173
14174                if (pathModel.getProcesses() != null) {
14175                        List<String> processIds = new ArrayList<String>();
14176                        for (Process process : pathModel.getProcesses()) {
14177                                processIds.add(String.valueOf(process.getId()));
14178                        }
14179                        pathElement.setProcessIds(processIds);
14180                }
14181
14182                pathElement.setUri(pathModel.getName());
14183
14184                dataflow.getPaths().add(pathElement);
14185
14186                List<TableColumn> columns = pathModel.getColumns();
14187
14188                for (int j = 0; j < columns.size(); j++) {
14189                        TableColumn columnModel = (TableColumn) columns.get(j);
14190                        column columnElement = new column();
14191                        columnElement.setId(String.valueOf(columnModel.getId()));
14192                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14193                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14194                                        + convertCoordinate(columnModel.getEndPosition()));
14195                        pathElement.getColumns().add(columnElement);
14196                }
14197
14198                TableRelationRows relationRows = pathModel.getRelationRows();
14199                if (relationRows.hasRelation()) {
14200                        column relationRowsElement = new column();
14201                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14202                        relationRowsElement.setName(relationRows.getName());
14203                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14204                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14205                                                + convertCoordinate(relationRows.getEndPosition()));
14206                        }
14207                        relationRowsElement.setSource("system");
14208                        pathElement.getColumns().add(relationRowsElement);
14209                }
14210        }
14211
14212        private void appendVariableModel(dataflow dataflow, Table variableModel) {
14213                table variableElement = new table();
14214                variableElement.setId(String.valueOf(variableModel.getId()));
14215                if (!SQLUtil.isEmpty(variableModel.getDatabase())) {
14216                        variableElement.setDatabase(variableModel.getDatabase());
14217                }
14218                if (!SQLUtil.isEmpty(variableModel.getSchema())) {
14219                        variableElement.setSchema(variableModel.getSchema());
14220                }
14221                variableElement.setServer(variableModel.getServer());
14222                variableElement.setName(variableModel.getName());
14223                variableElement.setType("variable");
14224                variableElement.setParent(variableModel.getParent());
14225                if (variableModel.getSubType() != null) {
14226                        variableElement.setSubType(variableModel.getSubType().name());
14227                }
14228
14229                if (variableModel.getStartPosition() != null && variableModel.getEndPosition() != null) {
14230                        variableElement.setCoordinate(convertCoordinate(variableModel.getStartPosition()) + ","
14231                                        + convertCoordinate(variableModel.getEndPosition()));
14232                }
14233                dataflow.getVariables().add(variableElement);
14234
14235                List<TableColumn> columns = variableModel.getColumns();
14236
14237                if (containStarColumn(columns)) {
14238                        for (TableColumn column : columns) {
14239                                if (column.getName().endsWith("*")) {
14240                                        for (TableColumn starElement : columns) {
14241                                                if (starElement == column) {
14242                                                        continue;
14243                                                }
14244                                                TObjectName columnObject = starElement.getColumnObject();
14245                                                column.bindStarLinkColumn(columnObject);
14246                                        }
14247//                                      column.setShowStar(false);
14248                                }
14249                        }
14250                }
14251
14252                for (int j = 0; j < columns.size(); j++) {
14253                        TableColumn columnModel = columns.get(j);
14254                        if (columnModel.hasStarLinkColumn()) {
14255                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
14256                                for (int k = 0; k < starLinkColumnList.size(); k++) {
14257                                        column columnElement = new column();
14258                                        columnElement.setId(columnModel.getId() + "_" + k);
14259                                        String columnName = starLinkColumnList.get(k);
14260                                        if (containStarColumn(columns, columnName)) {
14261                                                continue;
14262                                        }
14263                                        columnElement.setName(columnName);
14264                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14265                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14266                                                                + convertCoordinate(columnModel.getEndPosition()));
14267                                        }
14268                                        variableElement.getColumns().add(columnElement);
14269                                }
14270
14271                                if (columnModel.isShowStar()) {
14272                                        column columnElement = new column();
14273                                        columnElement.setId(String.valueOf(columnModel.getId()));
14274                                        columnElement.setName(columnModel.getName());
14275                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14276                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14277                                                                + convertCoordinate(columnModel.getEndPosition()));
14278                                        }
14279                                        variableElement.getColumns().add(columnElement);
14280                                }
14281
14282                        } else {
14283                                column columnElement = new column();
14284                                columnElement.setId(String.valueOf(columnModel.getId()));
14285                                columnElement.setName(columnModel.getName());
14286                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14287                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14288                                                        + convertCoordinate(columnModel.getEndPosition()));
14289                                }
14290                                variableElement.getColumns().add(columnElement);
14291                        }
14292                }
14293
14294                TableRelationRows relationRows = variableModel.getRelationRows();
14295                if (relationRows.hasRelation()) {
14296                        column relationRowsElement = new column();
14297                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14298                        relationRowsElement.setName(relationRows.getName());
14299                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14300                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14301                                                + convertCoordinate(relationRows.getEndPosition()));
14302                        }
14303                        relationRowsElement.setSource("system");
14304                        variableElement.getColumns().add(relationRowsElement);
14305                }
14306        }
14307
14308        private void appendCursorModel(dataflow dataflow, Table cursorModel) {
14309                table cursorElement = new table();
14310                cursorElement.setId(String.valueOf(cursorModel.getId()));
14311                if (!SQLUtil.isEmpty(cursorModel.getDatabase())) {
14312                        cursorElement.setDatabase(cursorModel.getDatabase());
14313                }
14314                if (!SQLUtil.isEmpty(cursorModel.getSchema())) {
14315                        cursorElement.setSchema(cursorModel.getSchema());
14316                }
14317                cursorElement.setServer(cursorModel.getServer());
14318                cursorElement.setName(cursorModel.getName());
14319                cursorElement.setType("variable");
14320                if (cursorElement.getSubType() != null) {
14321                        cursorElement.setSubType(cursorModel.getSubType().name());
14322                }
14323
14324                if (cursorModel.getStartPosition() != null && cursorModel.getEndPosition() != null) {
14325                        cursorElement.setCoordinate(convertCoordinate(cursorModel.getStartPosition()) + ","
14326                                        + convertCoordinate(cursorModel.getEndPosition()));
14327                }
14328                dataflow.getVariables().add(cursorElement);
14329
14330                List<TableColumn> columns = cursorModel.getColumns();
14331
14332                if (containStarColumn(columns)) {
14333                        for (TableColumn column : columns) {
14334                                if (column.getName().endsWith("*")) {
14335                                        for (TableColumn starElement : columns) {
14336                                                if (starElement == column) {
14337                                                        continue;
14338                                                }
14339                                                TObjectName columnObject = starElement.getColumnObject();
14340                                                column.bindStarLinkColumn(columnObject);
14341                                        }
14342                                        column.setShowStar(false);
14343                                }
14344                        }
14345                }
14346
14347                for (int j = 0; j < columns.size(); j++) {
14348                        TableColumn columnModel = (TableColumn) columns.get(j);
14349                        if (columnModel.hasStarLinkColumn()) {
14350                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
14351                                for (int k = 0; k < starLinkColumnList.size(); k++) {
14352                                        column columnElement = new column();
14353                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
14354                                        String columnName = starLinkColumnList.get(k);
14355                                        if (containStarColumn(columns, columnName)) {
14356                                                continue;
14357                                        }
14358                                        columnElement.setName(columnName);
14359                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14360                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14361                                                                + convertCoordinate(columnModel.getEndPosition()));
14362                                        }
14363                                        cursorElement.getColumns().add(columnElement);
14364                                }
14365
14366                                if (columnModel.isShowStar()) {
14367                                        column columnElement = new column();
14368                                        columnElement.setId(String.valueOf(columnModel.getId()));
14369                                        columnElement.setName(columnModel.getName());
14370                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14371                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14372                                                                + convertCoordinate(columnModel.getEndPosition()));
14373                                        }
14374                                        cursorElement.getColumns().add(columnElement);
14375                                }
14376
14377                        } else {
14378                                column columnElement = new column();
14379                                columnElement.setId(String.valueOf(columnModel.getId()));
14380                                columnElement.setName(columnModel.getName());
14381                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14382                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14383                                                        + convertCoordinate(columnModel.getEndPosition()));
14384                                }
14385                                cursorElement.getColumns().add(columnElement);
14386                        }
14387                }
14388
14389                TableRelationRows relationRows = cursorModel.getRelationRows();
14390                if (relationRows.hasRelation()) {
14391                        column relationRowsElement = new column();
14392                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14393                        relationRowsElement.setName(relationRows.getName());
14394                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14395                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14396                                                + convertCoordinate(relationRows.getEndPosition()));
14397                        }
14398                        relationRowsElement.setSource("system");
14399                        cursorElement.getColumns().add(relationRowsElement);
14400                }
14401        }
14402
14403        private void appendErrors(dataflow dataflow) {
14404                List<ErrorInfo> errorInfos = this.getErrorMessages();
14405                if (metadataErrors != null) {
14406                        for (int i = 0; i < metadataErrors.size(); i++) {
14407                                errorInfos.add(i, metadataErrors.get(i));
14408                        }
14409                }
14410
14411                for (int i = 0; i < errorInfos.size(); ++i) {
14412                        ErrorInfo errorInfo = errorInfos.get(i);
14413                        error error = new error();
14414                        if (!SQLUtil.isEmpty(errorInfo.getErrorMessage())) {
14415                                error.setErrorMessage(errorInfo.getErrorMessage());
14416                        }
14417                        if (!SQLUtil.isEmpty(errorInfo.getErrorType())) {
14418                                error.setErrorType(errorInfo.getErrorType());
14419                        }
14420                        if (errorInfo.getStartPosition() != null && errorInfo.getEndPosition() != null) {
14421                                error.setCoordinate(convertCoordinate(errorInfo.getStartPosition()) + ","
14422                                                + convertCoordinate(errorInfo.getEndPosition()));
14423                        }
14424                        if (!SQLUtil.isEmpty(errorInfo.getFileName())) {
14425                                error.setFile(errorInfo.getFileName());
14426                        }
14427                        if (errorInfo.getOriginStartPosition() != null && errorInfo.getOriginEndPosition() != null) {
14428                                error.setOriginCoordinate(errorInfo.getOriginStartPosition() + "," + errorInfo.getOriginEndPosition());
14429                        }
14430                        dataflow.getErrors().add(error);
14431                }
14432        }
14433
14434        private void appendOraclePackages(dataflow dataflow) {
14435                List<OraclePackage> packages = this.modelManager.getOraclePackageModels();
14436
14437                for (int i = 0; i < packages.size(); ++i) {
14438                        OraclePackage model = packages.get(i);
14439                        oraclePackage oraclePackage = new oraclePackage();
14440                        oraclePackage.setId(String.valueOf(model.getId()));
14441                        if (!SQLUtil.isEmpty(model.getDatabase())) {
14442                                oraclePackage.setDatabase(model.getDatabase());
14443                        }
14444                        if (!SQLUtil.isEmpty(model.getSchema())) {
14445                                oraclePackage.setSchema(model.getSchema());
14446                        }
14447                        oraclePackage.setServer(model.getServer());
14448                        oraclePackage.setName(model.getName());
14449                        if (model.getType() != null) {
14450                                oraclePackage.setType(model.getType().name().replace("sst", ""));
14451                        }
14452                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
14453                                oraclePackage.setCoordinate(
14454                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
14455                        }
14456
14457                        dataflow.getPackages().add(oraclePackage);
14458
14459                        List<Argument> arguments = model.getArguments();
14460
14461                        for (int j = 0; j < arguments.size(); ++j) {
14462                                Argument argumentModel = (Argument) arguments.get(j);
14463                                argument argumentElement = new argument();
14464                                argumentElement.setId(String.valueOf(argumentModel.getId()));
14465                                argumentElement.setName(argumentModel.getName());
14466                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
14467                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
14468                                                        + convertCoordinate(argumentModel.getEndPosition()));
14469                                }
14470
14471                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
14472                                argumentElement.setInout(argumentModel.getMode().name());
14473                                oraclePackage.getArguments().add(argumentElement);
14474                        }
14475
14476                        for (int j = 0; j < model.getProcedures().size(); j++) {
14477                                Procedure procedureModel = model.getProcedures().get(j);
14478                                procedure procedure = new procedure();
14479                                procedure.setId(String.valueOf(procedureModel.getId()));
14480                                if (!SQLUtil.isEmpty(procedureModel.getDatabase())) {
14481                                        procedure.setDatabase(procedureModel.getDatabase());
14482                                }
14483                                if (!SQLUtil.isEmpty(procedureModel.getSchema())) {
14484                                        procedure.setSchema(procedureModel.getSchema());
14485                                }
14486                                procedure.setServer(procedureModel.getServer());
14487                                procedure.setName(procedureModel.getName());
14488                                if (procedureModel.getType() != null) {
14489                                        procedure.setType(procedureModel.getType().name().replace("sst", ""));
14490                                }
14491                                if (procedureModel.getStartPosition() != null && procedureModel.getEndPosition() != null) {
14492                                        procedure.setCoordinate(convertCoordinate(procedureModel.getStartPosition()) + ","
14493                                                        + convertCoordinate(procedureModel.getEndPosition()));
14494                                }
14495
14496                                oraclePackage.getProcedures().add(procedure);
14497
14498                                List<Argument> procedureArguments = procedureModel.getArguments();
14499
14500                                for (int k = 0; k < procedureArguments.size(); ++k) {
14501                                        Argument argumentModel = (Argument) procedureArguments.get(k);
14502                                        argument argumentElement = new argument();
14503                                        argumentElement.setId(String.valueOf(argumentModel.getId()));
14504                                        argumentElement.setName(argumentModel.getName());
14505                                        if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
14506                                                argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
14507                                                                + convertCoordinate(argumentModel.getEndPosition()));
14508                                        }
14509
14510                                        argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
14511                                        argumentElement.setInout(argumentModel.getMode().name());
14512                                        procedure.getArguments().add(argumentElement);
14513                                }
14514                        }
14515                }
14516        }
14517
14518        private void appendProcedures(dataflow dataflow) {
14519                List<Procedure> procedures = this.modelManager.getProcedureModels();
14520
14521                for (int i = 0; i < procedures.size(); ++i) {
14522                        Procedure model = procedures.get(i);
14523                        if (model.getParentPackage() != null) {
14524                                continue;
14525                        }
14526                        procedure procedure = new procedure();
14527                        procedure.setId(String.valueOf(model.getId()));
14528                        if (!SQLUtil.isEmpty(model.getDatabase())) {
14529                                procedure.setDatabase(model.getDatabase());
14530                        }
14531                        if (!SQLUtil.isEmpty(model.getSchema())) {
14532                                procedure.setSchema(model.getSchema());
14533                        }
14534                        procedure.setServer(model.getServer());
14535                        procedure.setName(model.getName());
14536                        if (model.getType() != null) {
14537                                procedure.setType(model.getType().name().replace("sst", ""));
14538                        }
14539                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
14540                                procedure.setCoordinate(
14541                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
14542                        }
14543
14544                        dataflow.getProcedures().add(procedure);
14545
14546                        List<Argument> arguments = model.getArguments();
14547
14548                        for (int j = 0; j < arguments.size(); ++j) {
14549                                Argument argumentModel = (Argument) arguments.get(j);
14550                                argument argumentElement = new argument();
14551                                argumentElement.setId(String.valueOf(argumentModel.getId()));
14552                                argumentElement.setName(argumentModel.getName());
14553                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
14554                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
14555                                                        + convertCoordinate(argumentModel.getEndPosition()));
14556                                }
14557
14558                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
14559                                argumentElement.setInout(argumentModel.getMode().name());
14560                                procedure.getArguments().add(argumentElement);
14561                        }
14562                }
14563        }
14564
14565        private void appendProcesses(dataflow dataflow) {
14566                List<Process> processes = this.modelManager.getProcessModels();
14567
14568                for (int i = 0; i < processes.size(); ++i) {
14569                        Process model = processes.get(i);
14570                        process process = new process();
14571                        process.setId(String.valueOf(model.getId()));
14572                        if (!SQLUtil.isEmpty(model.getDatabase())) {
14573                                process.setDatabase(model.getDatabase());
14574                        }
14575                        if (!SQLUtil.isEmpty(model.getSchema())) {
14576                                process.setSchema(model.getSchema());
14577                        }
14578                        process.setServer(model.getServer());
14579                        process.setName(getProcessName(model));
14580                        if (!SQLUtil.isEmpty(model.getProcedureName())) {
14581                                process.setProcedureName(model.getProcedureName());
14582                        }
14583                        if (model.getProcedureId() != null) {
14584                                process.setProcedureId(String.valueOf(model.getProcedureId()));
14585                        }
14586                        if (!SQLUtil.isEmpty(model.getQueryHashId())) {
14587                                process.setQueryHashId(model.getQueryHashId());
14588                        }
14589                        if (model.getGspObject() != null) {
14590                                process.setType(model.getGspObject().sqlstatementtype.name());
14591                        }
14592                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
14593                                process.setCoordinate(
14594                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
14595                        }
14596                        if (model.getTransforms() != null && !model.getTransforms().isEmpty()) {
14597                                for (Transform transformItem : model.getTransforms()) {
14598                                        process.addTransform(transformItem);
14599                                }
14600                        }
14601                        dataflow.getProcesses().add(process);
14602                }
14603        }
14604
14605        private void appendTables(dataflow dataflow) {
14606                List<TTable> tables = modelManager.getBaseTables();
14607                Map<String, table> tableMap = new HashMap<String, table>();
14608                Set<Long> tableModelIds = new HashSet<Long>();
14609                for (int i = 0; i < tables.size(); i++) {
14610                        Object model = modelManager.getModel(tables.get(i));
14611                        if (model instanceof Table) {
14612                                Table tableModel = (Table) model;
14613                                if(tableModelIds.contains(tableModel.getId())) {
14614                                        continue;
14615                                }
14616                                else {
14617                                        tableModelIds.add(tableModel.getId());
14618                                }
14619                                if (tableModel.isView()) {
14620                                        continue;
14621                                }
14622                                if (tableModel.isStage()) {
14623                                        appendStageModel(dataflow, tableModel);
14624                                        continue;
14625                                }
14626                                if (tableModel.isSequence()) {
14627                                        appendSequenceModel(dataflow, tableModel);
14628                                        continue;
14629                                }
14630                                if (tableModel.isDataSource()) {
14631                                        appendDataSourceModel(dataflow, tableModel);
14632                                        continue;
14633                                }
14634                                if (tableModel.isDatabase()) {
14635                                        appendDatabaseModel(dataflow, tableModel);
14636                                        continue;
14637                                }
14638                                if (tableModel.isSchema()) {
14639                                        appendSchemaModel(dataflow, tableModel);
14640                                        continue;
14641                                }
14642                                if (tableModel.isStream()) {
14643                                        appendStreamModel(dataflow, tableModel);
14644                                        continue;
14645                                }
14646                                if (tableModel.isPath()) {
14647                                        appendPathModel(dataflow, tableModel);
14648                                        continue;
14649                                }
14650                                if (tableModel.isVariable() && !tableModel.isCursor()) {
14651                                        appendVariableModel(dataflow, tableModel);
14652                                        continue;
14653                                }
14654                                if (tableModel.isCursor()) {
14655                                        appendCursorModel(dataflow, tableModel);
14656                                        continue;
14657                                }
14658                                if (tableModel.isConstant()) {
14659                                        appendConstantModel(dataflow, tableModel);
14660                                        continue;
14661                                }
14662                                if (!tableIds.contains(tableModel.getId())) {
14663                                        appendTableModel(dataflow, tableModel, tableMap);
14664                                        tableIds.add(tableModel.getId());
14665                                }
14666                        } else if (model instanceof QueryTable) {
14667                                QueryTable queryTable = (QueryTable) model;
14668                                if (!tableIds.contains(queryTable.getId())) {
14669                                        appendResultSet(dataflow, queryTable);
14670                                        tableIds.add(queryTable.getId());
14671                                }
14672                        }
14673                }
14674
14675                List<Table> tableNames = modelManager.getTablesByName();
14676                tableNames.addAll(modelManager.getDropTables());
14677                
14678                for (int i = 0; i < tableNames.size(); i++) {
14679                        Table tableModel = tableNames.get(i);
14680                        if(tableModelIds.contains(tableModel.getId())) {
14681                                continue;
14682                        }
14683                        else {
14684                                tableModelIds.add(tableModel.getId());
14685                        }
14686                        if (tableModel.isView()) {
14687                                continue;
14688                        }
14689                        if (tableModel.isDatabase()) {
14690                                appendDatabaseModel(dataflow, tableModel);
14691                                continue;
14692                        }
14693                        if (tableModel.isSchema()) {
14694                                appendSchemaModel(dataflow, tableModel);
14695                                continue;
14696                        }
14697                        if (tableModel.isStage()) {
14698                                appendStageModel(dataflow, tableModel);
14699                                continue;
14700                        }
14701                        if (tableModel.isSequence()) {
14702                                appendSequenceModel(dataflow, tableModel);
14703                                continue;
14704                        }
14705                        if (tableModel.isDataSource()) {
14706                                appendDataSourceModel(dataflow, tableModel);
14707                                continue;
14708                        }
14709                        if (tableModel.isStream()) {
14710                                appendStreamModel(dataflow, tableModel);
14711                                continue;
14712                        }
14713                        if (tableModel.isPath()) {
14714                                appendPathModel(dataflow, tableModel);
14715                                continue;
14716                        }
14717                        if (tableModel.isVariable()) {
14718                                appendVariableModel(dataflow, tableModel);
14719                                continue;
14720                        }
14721                        if (tableModel.isCursor()) {
14722                                appendCursorModel(dataflow, tableModel);
14723                                continue;
14724                        }
14725                        if (tableModel.isConstant()) {
14726                                appendConstantModel(dataflow, tableModel);
14727                                continue;
14728                        }
14729                        if (!tableIds.contains(tableModel.getId())) {
14730                                appendTableModel(dataflow, tableModel, tableMap);
14731                                tableIds.add(tableModel.getId());
14732                        }
14733                }
14734        }
14735
14736        private void appendConstantModel(dataflow dataflow, Table tableModel) {
14737                table constantElement = new table();
14738                constantElement.setId(String.valueOf(tableModel.getId()));
14739                if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
14740                        constantElement.setDatabase(tableModel.getDatabase());
14741                }
14742                if (!SQLUtil.isEmpty(tableModel.getSchema())) {
14743                        constantElement.setSchema(tableModel.getSchema());
14744                }
14745                constantElement.setServer(tableModel.getServer());
14746                constantElement.setName(getConstantName(tableModel));
14747                constantElement.setType("constantTable");
14748
14749                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
14750                        constantElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
14751                                        + convertCoordinate(tableModel.getEndPosition()));
14752                }
14753                dataflow.getTables().add(constantElement);
14754
14755                List<TableColumn> columns = tableModel.getColumns();
14756                for (int j = 0; j < columns.size(); j++) {
14757                        TableColumn columnModel = (TableColumn) columns.get(j);
14758                        column columnElement = new column();
14759                        columnElement.setId(String.valueOf(columnModel.getId()));
14760                        columnElement.setName(columnModel.getName());
14761                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14762                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14763                                                + convertCoordinate(columnModel.getEndPosition()));
14764                        }
14765                        constantElement.getColumns().add(columnElement);
14766                }
14767        }
14768
14769        private void appendTableModel(dataflow dataflow, Table tableModel, Map<String, table> tableMap) {
14770                if(tableModel.getSubType() == SubType.unnest) {}
14771                table tableElement = new table();
14772                tableElement.setId(String.valueOf(tableModel.getId()));
14773                if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
14774                        tableElement.setDatabase(tableModel.getDatabase());
14775                }
14776                if (!SQLUtil.isEmpty(tableModel.getSchema())) {
14777                        tableElement.setSchema(tableModel.getSchema());
14778                }
14779                tableElement.setServer(tableModel.getServer());
14780                tableElement.setName(tableModel.getName());
14781                tableElement.setDisplayName(tableModel.getDisplayName());
14782                tableElement.setStarStmt(tableModel.getStarStmt());
14783                if(tableModel.isFromDDL()) {
14784                        tableElement.setFromDDL(String.valueOf(tableModel.isFromDDL()));
14785                }
14786
14787                if (tableModel.isPseudo()) {
14788                        tableElement.setType("pseudoTable");
14789                } else {
14790                        tableElement.setType("table");
14791                }
14792
14793                if (tableModel.getSubType() != null) {
14794                        if (tableModel.getSubType() == SubType.unnest) {
14795                                tableElement.setType(SubType.unnest.name());
14796                        }
14797                        else {
14798                                tableElement.setSubType(tableModel.getSubType().name());
14799                        }
14800                }
14801                if (tableModel.getParent() != null) {
14802                        tableElement.setParent(tableModel.getParent());
14803                }
14804                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
14805                        tableElement.setAlias(tableModel.getAlias());
14806                }
14807                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
14808            if(option.isTraceTablePosition()){
14809                for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:tableModel.getPositions()){
14810                    tableElement.appendCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
14811                }
14812            }
14813            else {
14814                tableElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
14815                    + convertCoordinate(tableModel.getEndPosition()));
14816            }
14817                }
14818                if (tableModel.getProcesses() != null) {
14819                        List<String> processIds = new ArrayList<String>();
14820                        for (Process process : tableModel.getProcesses()) {
14821                                processIds.add(String.valueOf(process.getId()));
14822                        }
14823                        tableElement.setProcessIds(processIds);
14824                }
14825
14826                table oldTableElement = null;
14827                String tableFullName = DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getQualifiedTableName(tableElement));
14828                
14829                if(tableMap.containsKey(tableFullName)) {
14830                        oldTableElement = tableMap.get(tableFullName);
14831                }
14832                else {
14833                        tableMap.put(tableFullName, tableElement);
14834                }
14835                
14836                if (tableModel.getSubType() == SubType.unnest) {
14837                        dataflow.getResultsets().add(tableElement);
14838                } else {
14839                        dataflow.getTables().add(tableElement);
14840                }
14841
14842                List<TableColumn> columns = tableModel.getColumns();
14843
14844                if (containStarColumn(columns)) {
14845                        for (TableColumn column : columns) {
14846                                if (column.getName().endsWith("*")) {
14847                                        for (TableColumn starElement : columns) {
14848                                                if (starElement == column) {
14849                                                        continue;
14850                                                }
14851                                                if (starElement.isNotBindStarLinkColumn()) {
14852                                                        continue;
14853                                                }
14854                                                TObjectName columnObject = starElement.getColumnObject();
14855                                                column.bindStarLinkColumn(columnObject);
14856                                        }
14857                                        if (tableModel.isCreateTable() && column.isExpandStar()) {
14858                                                column.setShowStar(false);
14859                                        }
14860                                }
14861                        }
14862                }
14863
14864                for (int j = 0; j < columns.size(); j++) {
14865                        TableColumn columnModel = columns.get(j);
14866
14867                        if(oldTableElement != null){
14868                                if(!CollectionUtil.isEmpty(oldTableElement.getColumns())){
14869                                        for(column oldColumnElement: oldTableElement.getColumns()){
14870                                                if(oldColumnElement.getName().equalsIgnoreCase(columnModel.getName())){
14871                                                        if(columnModel.getDataType() == null && oldColumnElement.getDataType() != null){
14872                                                                columnModel.setDataType(oldColumnElement.getDataType());
14873                                                        }
14874                                                        if(columnModel.getPrimaryKey() == null && oldColumnElement.isPrimaryKey() != null){
14875                                                                columnModel.setPrimaryKey(oldColumnElement.isPrimaryKey());
14876                                                        }
14877                                                        if(columnModel.getIndexKey() == null && oldColumnElement.isIndexKey() != null){
14878                                                                columnModel.setIndexKey(oldColumnElement.isIndexKey());
14879                                                        }
14880                                                        if(columnModel.getUnqiueKey() == null && oldColumnElement.isUnqiueKey() != null){
14881                                                                columnModel.setUnqiueKey(oldColumnElement.isUnqiueKey());
14882                                                        }
14883                                                        if(columnModel.getForeignKey() == null && oldColumnElement.isForeignKey() != null){
14884                                                                columnModel.setForeignKey(oldColumnElement.isForeignKey());
14885                                                        }
14886
14887                                                        if(oldColumnElement.getDataType() == null && columnModel.getDataType() != null){
14888                                                                oldColumnElement.setDataType(columnModel.getDataType());
14889                                                        }
14890                                                        if(oldColumnElement.isPrimaryKey() == null && columnModel.getPrimaryKey() != null){
14891                                                                oldColumnElement.setPrimaryKey(columnModel.getPrimaryKey());
14892                                                        }
14893                                                        if(oldColumnElement.isIndexKey() == null && columnModel.getIndexKey() != null){
14894                                                                oldColumnElement.setIndexKey(columnModel.getIndexKey());
14895                                                        }
14896                                                        if(oldColumnElement.isUnqiueKey() == null && columnModel.getUnqiueKey() != null){
14897                                                                oldColumnElement.setUnqiueKey(columnModel.getUnqiueKey());
14898                                                        }
14899                                                        if(oldColumnElement.isForeignKey() == null && columnModel.getForeignKey() != null){
14900                                                                oldColumnElement.setForeignKey(columnModel.getForeignKey());
14901                                                        }
14902                                                        break;
14903                                                }
14904                                        }
14905                                }
14906
14907                        }
14908
14909                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn() && !columnModel.isVariant()) {
14910                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
14911                                for (int k = 0; k < starLinkColumnList.size(); k++) {
14912                                        column columnElement = new column();
14913                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
14914                                        String columnName = starLinkColumnList.get(k);
14915                                        if (containStarColumn(columns, columnName)) {
14916                                                continue;
14917                                        }
14918                                        columnElement.setName(columnName);
14919                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14920                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14921                                                                + convertCoordinate(columnModel.getEndPosition()));
14922                                        }
14923                                        if (columnModel.getForeignKey()) {
14924                                                columnElement.setForeignKey(columnModel.getForeignKey());
14925                                        }
14926                                        if (columnModel.getIndexKey()) {
14927                                                columnElement.setIndexKey(columnModel.getIndexKey());
14928                                        }
14929                                        if (columnModel.getPrimaryKey()) {
14930                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
14931                                        }
14932                                        if (columnModel.getUnqiueKey()) {
14933                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
14934                                        }
14935                                        if (columnModel.getDataType() != null) {
14936                                                columnElement.setDataType(columnModel.getDataType());
14937                                        }
14938                                        tableElement.getColumns().add(columnElement);
14939                                }
14940                                if (columnModel.isShowStar()) {
14941                                        column columnElement = new column();
14942                                        columnElement.setId(String.valueOf(columnModel.getId()));
14943                                        columnElement.setName(columnModel.getName());
14944                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14945                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14946                                                                + convertCoordinate(columnModel.getEndPosition()));
14947                                        }
14948                                        if (columnModel.getForeignKey()) {
14949                                                columnElement.setForeignKey(columnModel.getForeignKey());
14950                                        }
14951                                        if (columnModel.getIndexKey()) {
14952                                                columnElement.setIndexKey(columnModel.getIndexKey());
14953                                        }
14954                                        if (columnModel.getPrimaryKey()) {
14955                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
14956                                        }
14957                                        if (columnModel.getUnqiueKey()) {
14958                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
14959                                        }
14960                                        if (columnModel.getDataType() != null) {
14961                                                columnElement.setDataType(columnModel.getDataType());
14962                                        }
14963                                        tableElement.getColumns().add(columnElement);
14964                                }
14965                        } else {
14966                                column columnElement = new column();
14967                                columnElement.setId(String.valueOf(columnModel.getId()));
14968                                columnElement.setName(columnModel.getName());
14969                                columnElement.setDisplayName(columnModel.getDisplayName());
14970                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14971                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14972                                                        + convertCoordinate(columnModel.getEndPosition()));
14973                                }
14974                                if (columnModel.isPseduo()) {
14975                                        columnElement.setSource("system");
14976                                }
14977                                if (columnModel.getForeignKey()) {
14978                                        columnElement.setForeignKey(columnModel.getForeignKey());
14979                                }
14980                                if (columnModel.getIndexKey()) {
14981                                        columnElement.setIndexKey(columnModel.getIndexKey());
14982                                }
14983                                if (columnModel.getPrimaryKey()) {
14984                                        columnElement.setPrimaryKey(columnModel.getPrimaryKey());
14985                                }
14986                                if (columnModel.getUnqiueKey()) {
14987                                        columnElement.setUnqiueKey(columnModel.getUnqiueKey());
14988                                }
14989                                if (columnModel.getDataType() != null) {
14990                                        columnElement.setDataType(columnModel.getDataType());
14991                                }
14992                                tableElement.getColumns().add(columnElement);
14993                        }
14994                }
14995
14996                TableRelationRows relationRows = tableModel.getRelationRows();
14997                if (relationRows.hasRelation()) {
14998                        column relationRowsElement = new column();
14999                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
15000                        relationRowsElement.setName(relationRows.getName());
15001                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
15002                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
15003                                                + convertCoordinate(relationRows.getEndPosition()));
15004                        }
15005                        relationRowsElement.setSource("system");
15006                        tableElement.getColumns().add(relationRowsElement);
15007                }
15008        }
15009
15010        private boolean containStarColumn(List<TableColumn> columns, String qualifiedColumnName) {
15011                for (TableColumn tableColumn : columns) {
15012                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
15013                                return true;
15014                        }
15015                }
15016                return false;
15017        }
15018
15019        private TableColumn searchTableColumn(List<TableColumn> columns, String qualifiedColumnName) {
15020                for (TableColumn tableColumn : columns) {
15021                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
15022                                return tableColumn;
15023                        }
15024                }
15025                return null;
15026        }
15027
15028        private boolean containStarColumn(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
15029                for (RelationshipElement element : elements) {
15030                        if (element.getElement() instanceof TableColumn) {
15031                                if (DlineageUtil.getIdentifierNormalColumnName(((TableColumn) element.getElement()).getName())
15032                                                .equals(qualifiedColumnName)) {
15033                                        return true;
15034                                }
15035                        } else if (element.getElement() instanceof ResultColumn) {
15036                                if (DlineageUtil.getIdentifierNormalColumnName(((ResultColumn) element.getElement()).getName())
15037                                                .equals(qualifiedColumnName)) {
15038                                        return true;
15039                                }
15040                        }
15041                }
15042                return false;
15043        }
15044
15045        /**
15046         * Returns true if there is a non-star source element matching the column name
15047         * whose parent (table/resultset) does NOT also contribute a star source in
15048         * the same relationship. This identifies "definitive" explicit sources like
15049         * subquery columns (e.g., "coalesce(...) as col_a") vs incidental references
15050         * (e.g., "aTab.id" where aTab.* is also a source).
15051         */
15052        private boolean hasDefinitiveNonStarSource(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
15053                // First, collect parent IDs of all star (*) sources
15054                Set<Long> starSourceParentIds = new HashSet<Long>();
15055                for (RelationshipElement<?> element : elements) {
15056                        if (element.getElement() instanceof TableColumn) {
15057                                TableColumn tc = (TableColumn) element.getElement();
15058                                if ("*".equals(tc.getName()) && tc.getTable() != null) {
15059                                        starSourceParentIds.add(tc.getTable().getId());
15060                                }
15061                        } else if (element.getElement() instanceof ResultColumn) {
15062                                ResultColumn rc = (ResultColumn) element.getElement();
15063                                if ("*".equals(rc.getName()) && rc.getResultSet() != null) {
15064                                        starSourceParentIds.add(rc.getResultSet().getId());
15065                                }
15066                        }
15067                }
15068
15069                // Then check if any non-star source matching the column name has a parent
15070                // that does NOT also have a star source
15071                for (RelationshipElement<?> element : elements) {
15072                        if (element.getElement() instanceof TableColumn) {
15073                                TableColumn tc = (TableColumn) element.getElement();
15074                                if (!"*".equals(tc.getName())
15075                                                && DlineageUtil.getIdentifierNormalColumnName(tc.getName()).equals(qualifiedColumnName)) {
15076                                        if (tc.getTable() != null && !starSourceParentIds.contains(tc.getTable().getId())) {
15077                                                return true;
15078                                        }
15079                                }
15080                        } else if (element.getElement() instanceof ResultColumn) {
15081                                ResultColumn rc = (ResultColumn) element.getElement();
15082                                if (!"*".equals(rc.getName())
15083                                                && DlineageUtil.getIdentifierNormalColumnName(rc.getName()).equals(qualifiedColumnName)) {
15084                                        if (rc.getResultSet() != null && !starSourceParentIds.contains(rc.getResultSet().getId())) {
15085                                                return true;
15086                                        }
15087                                }
15088                        }
15089                }
15090                return false;
15091        }
15092
15093        private void analyzeSelectStmt(TSelectSqlStatement stmt) {
15094                if (!accessedSubqueries.contains(stmt)) {
15095                        accessedSubqueries.add(stmt);
15096                } else {
15097                        if (modelManager.getModel(stmt) != null) {
15098                                return;
15099                        }
15100                }
15101
15102                if (stmt.getParentStmt() == null && stmt.getIntoClause() == null && stmt.getIntoTableClause() == null) {
15103                        if(option.isIgnoreTopSelect() && (option.isIgnoreRecordSet() || option.isSimpleOutput())){
15104                                if(option.getAnalyzeMode() == null || option.getAnalyzeMode() == AnalyzeMode.dataflow){
15105                                        return;
15106                                }
15107                        }
15108                }
15109
15110                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
15111
15112                        // Iteratively analyze all descendant UNION branches before processing this node.
15113                        // Uses iterative post-order traversal to preserve the original left-right-self
15114                        // processing order, avoiding StackOverflow with deeply nested UNION trees.
15115                        {
15116                                Deque<TSelectSqlStatement> stack1 = new ArrayDeque<>();
15117                                List<TSelectSqlStatement> postOrder = new ArrayList<>();
15118                                stack1.push(stmt);
15119                                while (!stack1.isEmpty()) {
15120                                        TSelectSqlStatement node = stack1.pop();
15121                                        postOrder.add(node);
15122                                        if (node.getSetOperatorType() != ESetOperatorType.none) {
15123                                                // Push left first, then right, so that after reversal left comes first
15124                                                if (node.getLeftStmt() != null) stack1.push(node.getLeftStmt());
15125                                                if (node.getRightStmt() != null) stack1.push(node.getRightStmt());
15126                                        }
15127                                }
15128                                Collections.reverse(postOrder);
15129                                // Process all descendants in post-order, skip stmt itself (processed below)
15130                                for (int pi = 0; pi < postOrder.size() - 1; pi++) {
15131                                        TSelectSqlStatement node = postOrder.get(pi);
15132                                        if (!accessedStatements.contains(node)) {
15133                                                accessedStatements.add(node);
15134                                                analyzeSelectStmt(node);
15135                                        }
15136                                }
15137                        }
15138
15139                        stmtStack.push(stmt);
15140                        SelectSetResultSet resultSet = modelFactory.createSelectSetResultSet(stmt);
15141
15142                        ResultSet leftResultSetModel = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
15143                        if (leftResultSetModel != null && leftResultSetModel != resultSet
15144                                        && !leftResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15145                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15146                                impactRelation.setEffectType(EffectType.select);
15147                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15148                                                leftResultSetModel.getRelationRows()));
15149                                impactRelation.setTarget(
15150                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
15151                        }
15152                        
15153                        ResultSet rightResultSetModel = (ResultSet) modelManager.getModel(stmt.getRightStmt());
15154                        if (rightResultSetModel != null && rightResultSetModel != resultSet
15155                                        && !rightResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15156                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15157                                impactRelation.setEffectType(EffectType.select);
15158                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15159                                                rightResultSetModel.getRelationRows()));
15160                                impactRelation.setTarget(
15161                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
15162                        }
15163                        
15164                        if ((leftResultSetModel != null && leftResultSetModel.isDetermined())
15165                                        || (rightResultSetModel != null && rightResultSetModel.isDetermined())) {
15166                                resultSet.setDetermined(true);
15167                        }
15168                        
15169                        if (resultSet.getColumns() == null || resultSet.getColumns().isEmpty()) {
15170                                if (getResultColumnList(stmt.getLeftStmt()) != null) {
15171                                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
15172                                } else if (getResultColumnList(stmt.getRightStmt()) != null) {
15173                                        createSelectSetResultColumns(resultSet, stmt.getRightStmt());
15174                                }
15175                        }
15176
15177                        List<ResultColumn> columns = resultSet.getColumns();
15178                        for (int i = 0; i < columns.size(); i++) {
15179                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15180                                relation.setEffectType(EffectType.select);
15181                                relation.setTarget(new ResultColumnRelationshipElement(columns.get(i)));
15182
15183                                if (!stmt.getLeftStmt().isCombinedQuery()) {
15184                                        ResultSet sourceResultSet = (ResultSet) modelManager
15185                                                        .getModel(stmt.getLeftStmt().getResultColumnList());
15186                                        if (sourceResultSet!=null && sourceResultSet.getColumns().size() > i) {
15187                                                if (columns.get(i).getName().endsWith("*")) {
15188                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
15189                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15190                                                        }
15191                                                } else {
15192                                                        relation.addSource(
15193                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15194                                                }
15195                                        }
15196                                } else {
15197                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
15198                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
15199                                                if (columns.get(i).getName().endsWith("*")) {
15200                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
15201                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15202                                                        }
15203                                                } else {
15204                                                        relation.addSource(
15205                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15206                                                }
15207                                        }
15208                                }
15209
15210                                if (!stmt.getRightStmt().isCombinedQuery()) {
15211                                        ResultSet sourceResultSet = (ResultSet) modelManager
15212                                                        .getModel(stmt.getRightStmt().getResultColumnList());
15213                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
15214                                                if (columns.get(i).getName().endsWith("*")) {
15215                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
15216                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15217                                                        }
15218                                                } else {
15219                                                        relation.addSource(
15220                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15221                                                }
15222                                        } else if (sourceResultSet != null) {
15223                                                for (ResultColumn column : sourceResultSet.getColumns()) {
15224                                                        if (column.hasStarLinkColumn()) {
15225                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15226                                                        }
15227                                                }
15228                                        }
15229                                } else {
15230                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getRightStmt());
15231                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
15232                                                relation.addSource(new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15233                                        } else if (sourceResultSet != null) {
15234                                                for (ResultColumn column : sourceResultSet.getColumns()) {
15235                                                        if (column.hasStarLinkColumn()) {
15236                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15237                                                        }
15238                                                }
15239                                        }
15240                                }
15241                        }
15242                        
15243                        analyzeSelectIntoClause(stmt);
15244                        
15245                        stmtStack.pop();
15246                } else {
15247
15248                        // handle hive stmt, issue_id I3SGZB
15249                        if (stmt.getHiveBodyList() != null && stmt.getHiveBodyList().size() > 0) {
15250                                stmtStack.push(stmt);
15251                                hiveFromTables = stmt.tables;
15252                                if (hiveFromTables != null) {
15253                                        for (int i = 0; i < hiveFromTables.size(); i++) {
15254                                                modelFactory.createTable(hiveFromTables.getTable(i));
15255                                        }
15256                                }
15257                                for (int i = 0; i < stmt.getHiveBodyList().size(); i++) {
15258                                        analyzeCustomSqlStmt(stmt.getHiveBodyList().get(i));
15259                                }
15260                                stmtStack.pop();
15261                                return;
15262                        }
15263                        
15264                        if (stmt.getTransformClause() != null) {
15265                                analyzeHiveTransformClause(stmt, stmt.getTransformClause());
15266                                return;
15267                        }
15268
15269                        if (stmt.getResultColumnList() == null) {
15270                                return;
15271                        }
15272
15273                        stmtStack.push(stmt);
15274
15275                        TTableList fromTables = stmt.tables;
15276                        if ((fromTables == null || fromTables.size() == 0)
15277                                        && (hiveFromTables != null && hiveFromTables.size() > 0)) {
15278                                fromTables = hiveFromTables;
15279                        }
15280
15281                        for (int i = 0; i < fromTables.size(); i++) {
15282                                TTable table = fromTables.getTable(i);
15283                                if (table.getLateralViewList() != null && !table.getLateralViewList().isEmpty()) {
15284                                        analyzeTableSubquery(table);
15285                                        analyzeLateralView(stmt, table, table.getLateralViewList());
15286                                        stmtStack.pop();
15287                                        return;
15288                                }
15289                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvpresto) {
15290                                        analyzeTableSubquery(table);
15291                                        analyzePrestoUnnest(stmt, table);
15292                                        stmtStack.pop();
15293                                        return;
15294                                }
15295                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvbigquery) {
15296                                        analyzeBigQueryUnnest(stmt, table);
15297                                }
15298                        }
15299                        
15300                        //用来获取 pivotedTable 对应的columns
15301                        TPivotedTable pivotedTable = null; 
15302                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
15303                                for (TJoin join : stmt.getJoins()) {
15304                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
15305                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
15306                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
15307                                                        pivotedTable = join.getTable().getPivotedTable();
15308                                                } else {
15309                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
15310                                                        pivotedTable = join.getTable().getPivotedTable();
15311                                                }
15312                                        }
15313                                }
15314                        }
15315
15316                        for (int i = 0; i < fromTables.size(); i++) {
15317                                TTable table = fromTables.getTable(i);
15318                                // Handle TABLE(func()) which has tableType=tableExpr and funcCall=null
15319                                if (table.getTableType() == ETableSource.tableExpr && table.getFuncCall() == null && pipelinedAnalyzer != null) {
15320                                        try {
15321                                                pipelinedAnalyzer.tryStitchTableExpr(table);
15322                                        } catch (Exception e) {
15323                                                // Don't let pipelined stitching failure break main flow
15324                                        }
15325                                }
15326
15327                                if (table.getFuncCall() != null) {
15328                                        TFunctionCall functionCall = table.getFuncCall();
15329                                        Procedure callee = modelManager.getProcedureByName(
15330                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
15331                                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
15332                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
15333                                                callee = modelManager.getProcedureByName(
15334                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
15335                                        }
15336
15337                                        if(callee!=null) {
15338                                                if (callee.getArguments() != null) {
15339                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
15340                                                                Argument argument = callee.getArguments().get(j);
15341                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
15342                                                                if(variable!=null) {
15343                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
15344                                                                                Transform transform = new Transform();
15345                                                                                transform.setType(Transform.FUNCTION);
15346                                                                                transform.setCode(functionCall);
15347                                                                                variable.getColumns().get(0).setTransform(transform);
15348                                                                        }
15349                                                                        Process process = modelFactory.createProcess(functionCall);
15350                                                                        variable.addProcess(process);
15351                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
15352                                                                }
15353                                                        }
15354                                                }
15355                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
15356                                                                .getIdentifierNormalTableName(table.getFuncCall().getFunctionName().toString()));
15357                                                if (functionTableModelObjs != null) {
15358                                                        modelManager.bindModel(table, functionTableModelObjs.iterator().next());
15359                                                }
15360                                                // Try pipelined function stitching
15361                                                if (pipelinedAnalyzer != null) {
15362                                                        try {
15363                                                                pipelinedAnalyzer.tryStitchCallSite(table, functionCall);
15364                                                        } catch (Exception e) {
15365                                                                // Don't let pipelined stitching failure break main flow
15366                                                        }
15367                                                }
15368                                                continue;
15369                                        } else {
15370                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
15371                                                                .getIdentifierNormalTableName(table.getFuncCall().getFunctionName().toString()));
15372                                                if (functionTableModelObjs == null) {
15373//                                                      Procedure procedure = modelManager.getProcedureByName(DlineageUtil
15374//                                                                      .getIdentifierNormalTableName(table.getFuncCall().getFunctionName().toString()));
15375//                                                      if (procedure != null) {
15376                                                                createFunction(table.getFuncCall());
15377                                                                continue;
15378//                                                      }
15379                                                }
15380                                                if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof Table) {
15381                                                        Table functionTableModel = (Table) functionTableModelObjs.iterator().next();
15382                                                        if (functionTableModel.getColumns() != null) {
15383                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
15384                                                                if (functionTable == functionTableModel)
15385                                                                        continue;
15386                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
15387                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
15388                                                                                        functionTableModel.getColumns().get(j).getColumnObject(), true);
15389                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15390                                                                        relation.setEffectType(EffectType.select);
15391                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
15392                                                                        relation.addSource(
15393                                                                                        new TableColumnRelationshipElement(functionTableModel.getColumns().get(j)));
15394                                                                }
15395                                                        }
15396                                                } else if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
15397                                                        ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
15398                                                        if (functionTableModel.getColumns() != null) {
15399                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
15400                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
15401                                                                        TParseTreeNode columnObj = functionTableModel.getColumns().get(j).getColumnObject();
15402                                                                        if (columnObj instanceof TObjectName) {
15403                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
15404                                                                                                (TObjectName) columnObj, true);
15405                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15406                                                                                relation.setEffectType(EffectType.select);
15407                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
15408                                                                                relation.addSource(new ResultColumnRelationshipElement(
15409                                                                                                functionTableModel.getColumns().get(j)));
15410                                                                        } else if (columnObj instanceof TResultColumn) {
15411                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
15412                                                                                                (TResultColumn) columnObj);
15413                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15414                                                                                relation.setEffectType(EffectType.select);
15415                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
15416                                                                                relation.addSource(new ResultColumnRelationshipElement(
15417                                                                                                functionTableModel.getColumns().get(j)));
15418                                                                        }
15419                                                                }
15420                                                        }
15421                                                }
15422                                                // Try pipelined function stitching for unresolved function calls
15423                                                if (pipelinedAnalyzer != null) {
15424                                                        try {
15425                                                                pipelinedAnalyzer.tryStitchCallSite(table, table.getFuncCall());
15426                                                        } catch (Exception e) {
15427                                                                // Don't let pipelined stitching failure break main flow
15428                                                        }
15429                                                }
15430                                        }
15431                                }
15432
15433                                if (table.getPartitionExtensionClause() != null
15434                                                && table.getPartitionExtensionClause().getKeyValues() != null) {
15435                                        TExpressionList values = table.getPartitionExtensionClause().getKeyValues();
15436                                        Table tableModel = modelFactory.createTable(table);
15437                                        for (TExpression value : values) {
15438                                                if (value.getExpressionType() == EExpressionType.simple_constant_t) {
15439                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15440                                                        impactRelation.setEffectType(EffectType.select);
15441                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
15442                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, value.getConstantOperand());
15443                                                        impactRelation.addSource(new TableColumnRelationshipElement(constantColumn));
15444                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
15445                                                                        tableModel.getRelationRows()));
15446                                                }
15447                                        }
15448                                }
15449
15450                                if (table.getSubquery() != null) {
15451                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15452                                        TSelectSqlStatement subquery = table.getSubquery();
15453                                        analyzeSelectStmt(subquery);
15454
15455                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
15456                                        if (resultSetModel != null && resultSetModel.isDetermined()) {
15457                                                queryTable.setDetermined(true);
15458                                        }
15459
15460                                        if (resultSetModel != null && resultSetModel != queryTable
15461                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15462                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15463                                                impactRelation.setEffectType(EffectType.select);
15464                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15465                                                                resultSetModel.getRelationRows()));
15466                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15467                                                                queryTable.getRelationRows()));
15468                                        }
15469
15470                                        if (resultSetModel != null && resultSetModel != queryTable
15471                                                        && queryTable.getTableObject().getAliasClause() != null
15472                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
15473                                                for (int j = 0; j < queryTable.getColumns().size()
15474                                                                && j < resultSetModel.getColumns().size(); j++) {
15475                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15476                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
15477
15478                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
15479                                                        queryRalation.setEffectType(EffectType.select);
15480                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15481                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15482                                                }
15483                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15484                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
15485                                                                .getModel(subquery);
15486                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15487                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15488                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
15489                                                                        sourceColumn);
15490                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15491                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
15492                                                        }
15493                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15494                                                        selectSetRalation.setEffectType(EffectType.select);
15495                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15496                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15497                                                }
15498                                        }
15499                                } else if (table.getOutputMerge() != null) {
15500                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15501                                        TMergeSqlStatement subquery = table.getOutputMerge();
15502                                        analyzeMergeStmt(subquery);
15503
15504                                        for (TResultColumn column : subquery.getOutputClause().getSelectItemList()) {
15505                                                modelFactory.createResultColumn(queryTable, column);
15506                                                analyzeResultColumn(column, EffectType.select);
15507                                        }
15508
15509                                        ResultSet resultSetModel = (ResultSet) modelManager
15510                                                        .getModel(subquery.getOutputClause().getSelectItemList());
15511
15512                                        if (resultSetModel != null && resultSetModel != queryTable
15513                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15514                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15515                                                impactRelation.setEffectType(EffectType.select);
15516                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15517                                                                resultSetModel.getRelationRows()));
15518                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15519                                                                queryTable.getRelationRows()));
15520                                        }
15521
15522                                        if (resultSetModel != null && resultSetModel != queryTable
15523                                                        && queryTable.getTableObject().getAliasClause() != null
15524                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
15525                                                for (int j = 0; j < queryTable.getColumns().size()
15526                                                                && j < resultSetModel.getColumns().size(); j++) {
15527                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15528                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
15529
15530                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
15531                                                        queryRalation.setEffectType(EffectType.select);
15532                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15533                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15534                                                }
15535                                        }
15536                                } else if (table.getTableExpr() != null && table.getTableExpr().getSubQuery() != null) {
15537                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15538                                        TSelectSqlStatement subquery = table.getTableExpr().getSubQuery();
15539                                        analyzeSelectStmt(subquery);
15540
15541                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
15542
15543                                        if (resultSetModel != null && resultSetModel != queryTable
15544                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15545                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15546                                                impactRelation.setEffectType(EffectType.select);
15547                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15548                                                                resultSetModel.getRelationRows()));
15549                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15550                                                                queryTable.getRelationRows()));
15551                                        }
15552
15553                                        if (resultSetModel != null && resultSetModel != queryTable
15554                                                        && queryTable.getTableObject().getAliasClause() != null) {
15555                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
15556                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15557                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
15558                                                                        sourceColumn);
15559
15560                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
15561                                                        queryRalation.setEffectType(EffectType.select);
15562                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15563                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15564                                                }
15565                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15566                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
15567                                                                .getModel(subquery);
15568                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15569                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15570                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
15571                                                                        sourceColumn);
15572                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15573                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
15574                                                        }
15575                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15576                                                        selectSetRalation.setEffectType(EffectType.select);
15577                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15578                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15579                                                }
15580                                        }
15581                                } else if (table.getCTE() != null) {
15582                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15583
15584                                        TObjectNameList cteColumns = table.getCTE().getColumnList();
15585                                        if (cteColumns != null) {
15586                                                for (int j = 0; j < cteColumns.size(); j++) {
15587                                                        modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
15588                                                }
15589                                                queryTable.setDetermined(true);
15590                                        }
15591                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
15592                                        if (subquery != null && !stmtStack.contains(subquery) && subquery.getResultColumnList() != null) {
15593                                                analyzeSelectStmt(subquery);
15594
15595                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
15596                                                if (resultSetModel != null && resultSetModel != queryTable
15597                                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15598                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15599                                                        impactRelation.setEffectType(EffectType.select);
15600                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15601                                                                        resultSetModel.getRelationRows()));
15602                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15603                                                                        queryTable.getRelationRows()));
15604                                                }
15605
15606                                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15607                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
15608                                                                        .getModel(subquery);
15609                                                        int x = 0;
15610                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15611                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15612                                                                ResultColumn targetColumn = null;
15613                                                                if (cteColumns != null) {
15614                                                                        if (queryTable.getColumns().size() <= j) {
15615                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
15616                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
15617                                                                                                        || queryTable.getColumns().get(k).getName()
15618                                                                                                                        .equals(sourceColumn.getAlias())) {
15619                                                                                                targetColumn = queryTable.getColumns().get(k);
15620                                                                                        }
15621                                                                                }
15622                                                                        } else {
15623                                                                                if (x < j) {
15624                                                                                        x = j;
15625                                                                                }
15626                                                                                targetColumn = queryTable.getColumns().get(x);
15627                                                                                x++;
15628                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
15629                                                                                                && x < cteColumns.size()) {
15630                                                                                        j--;
15631                                                                                }
15632                                                                        }
15633                                                                } else {
15634                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
15635                                                                }
15636                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15637                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
15638                                                                }
15639                                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15640                                                                selectSetRalation.setEffectType(EffectType.select);
15641                                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15642                                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15643                                                        }
15644                                                        if (!queryTable.isDetermined()) {
15645                                                                queryTable.setDetermined(selectSetResultSetModel.isDetermined());
15646                                                        }
15647                                                } else {
15648                                                        int x = 0;
15649                                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
15650                                                                ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15651                                                                ResultColumn targetColumn = null;
15652                                                                if (cteColumns != null) {
15653                                                                        if (queryTable.getColumns().size() <= j) {
15654                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
15655                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
15656                                                                                                        || queryTable.getColumns().get(k).getName()
15657                                                                                                                        .equals(sourceColumn.getAlias())) {
15658                                                                                                targetColumn = queryTable.getColumns().get(k);
15659                                                                                        }
15660                                                                                }
15661                                                                        } else {
15662                                                                                if (x < j) {
15663                                                                                        x = j;
15664                                                                                }
15665                                                                                targetColumn = queryTable.getColumns().get(x);
15666                                                                                x++;
15667                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
15668                                                                                                && x < cteColumns.size()) {
15669                                                                                        j--;
15670                                                                                }
15671                                                                        }
15672                                                                } else {
15673                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
15674                                                                }
15675                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15676                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
15677                                                                }
15678                                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15679                                                                selectSetRalation.setEffectType(EffectType.select);
15680                                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15681                                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15682                                                        }
15683                                                        if (!queryTable.isDetermined()) {
15684                                                                queryTable.setDetermined(resultSetModel.isDetermined());
15685                                                        }
15686                                                }
15687                                        } else if (table.getCTE().getUpdateStmt() != null) {
15688                                                analyzeCustomSqlStmt(table.getCTE().getUpdateStmt());
15689                                        } else if (table.getCTE().getInsertStmt() != null) {
15690                                                analyzeCustomSqlStmt(table.getCTE().getInsertStmt());
15691                                        } else if (table.getCTE().getDeleteStmt() != null) {
15692                                                analyzeCustomSqlStmt(table.getCTE().getDeleteStmt());
15693                                        }
15694                                } else if (table.getTableType().name().startsWith("open")) {
15695                                        continue;
15696                                } else if (table.getTableType() == ETableSource.jsonTable) {
15697                                        Table functionTable = modelFactory.createJsonTable(table);
15698                                        TJsonTable jsonTable = table.getJsonTable();
15699                                        TColumnDefinitionList definitions = jsonTable.getColumnDefinitions();
15700                                        if (definitions != null) {
15701                                                for (int j = 0; j < definitions.size(); j++) {
15702                            TColumnDefinitionList nestDefinitions = definitions.getColumn(j).getNestedTableColumns();
15703                                                        if(nestDefinitions!=null) {
15704                                                                for(int k=0;k<nestDefinitions.size();k++){
15705                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
15706                                            nestDefinitions.getColumn(k).getColumnName(), true);
15707                                    if (nestDefinitions.getColumn(k).getColumnPath() != null) {
15708                                        column.setDisplayName(
15709                                                column.getName() + ":" + nestDefinitions.getColumn(k).getColumnPath());
15710                                    }
15711                                                                }
15712                                                        }
15713                                                        else {
15714                                TableColumn column = modelFactory.createTableColumn(functionTable,
15715                                        definitions.getColumn(j).getColumnName(), true);
15716                                if (definitions.getColumn(j).getColumnPath() != null) {
15717                                    column.setDisplayName(
15718                                            column.getName() + ":" + definitions.getColumn(j).getColumnPath());
15719                                }
15720                            }
15721                                                }
15722                                        } else {
15723                                                TObjectName keyColumn = new TObjectName();
15724                                                keyColumn.setString("key");
15725                                                modelFactory.createJsonTableColumn(functionTable, keyColumn);
15726                                                TObjectName valueColumn = new TObjectName();
15727                                                valueColumn.setString("value");
15728                                                modelFactory.createJsonTableColumn(functionTable, valueColumn);
15729                                                TObjectName typeColumn = new TObjectName();
15730                                                typeColumn.setString("type");
15731                                                modelFactory.createJsonTableColumn(functionTable, typeColumn);
15732                                        }
15733
15734                                        functionTable.setCreateTable(true);
15735                                        functionTable.setSubType(SubType.function);
15736                                        modelManager.bindCreateModel(table, functionTable);
15737
15738                                        if (jsonTable.getJsonExpression() == null) {
15739                                                ErrorInfo errorInfo = new ErrorInfo();
15740                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
15741                                                errorInfo.setErrorMessage("Can't handle the json table: " + jsonTable.toString());
15742                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(jsonTable.getStartToken().lineNo,
15743                                                                jsonTable.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
15744                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(jsonTable.getEndToken().lineNo,
15745                                                                jsonTable.getEndToken().columnNo + jsonTable.getEndToken().getAstext().length(),
15746                                                                ModelBindingManager.getGlobalHash()));
15747                                                errorInfo.fillInfo(this);
15748                                                errorInfos.add(errorInfo);
15749                                        } else {
15750                                                String jsonName = jsonTable.getJsonExpression().toString();
15751                                                if (!jsonName.startsWith("@")) {
15752                                                        columnsInExpr visitor = new columnsInExpr();
15753                                                        jsonTable.getJsonExpression().inOrderTraverse(visitor);
15754                                                        List<TObjectName> objectNames = visitor.getObjectNames();
15755                                                        List<TParseTreeNode> functions = visitor.getFunctions();
15756                                                        List<TParseTreeNode> constants = visitor.getConstants();
15757                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15758
15759                                                        for (int j = 0; j < functionTable.getColumns().size(); j++) {
15760                                                                TableColumn tableColumn = functionTable.getColumns().get(j);
15761                                                                if (functions != null && !functions.isEmpty()) {
15762                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
15763                                                                }
15764                                                                if (subquerys != null && !subquerys.isEmpty()) {
15765                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
15766                                                                }
15767                                                                if (objectNames != null && !objectNames.isEmpty()) {
15768                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
15769                                                                }
15770                                                                if (constants != null && !constants.isEmpty()) {
15771                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
15772                                                                                        functions);
15773                                                                }
15774                                                        }
15775                                                } else {
15776                                                        TStatementList stmts = stmt.getGsqlparser().getSqlstatements();
15777                                                        for (int j = 0; j < stmts.size(); j++) {
15778                                                                TCustomSqlStatement item = stmts.get(j);
15779                                                                if (item instanceof TMssqlDeclare) {
15780                                                                        if (analyzeMssqlJsonDeclare((TMssqlDeclare) item, jsonName, functionTable)) {
15781                                                                                break;
15782                                                                        }
15783                                                                }
15784                                                        }
15785                                                }
15786                                        }
15787                                } else if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) {
15788                                        if (table.getTableType() == ETableSource.rowList && table.getRowList() != null
15789                                                        && table.getRowList().size() > 0) {
15790                                                Table tableModel = modelFactory.createTable(table);
15791                                                for (int j = 0; j < table.getRowList().size(); j++) {
15792                                                        TMultiTarget rowList = table.getRowList().getMultiTarget(j);
15793                                                        for (int k = 0; k < rowList.getColumnList().size(); k++) {
15794                                                                TResultColumn column = rowList.getColumnList().getResultColumn(k);
15795                                                                if (column.getFieldAttr() == null)
15796                                                                        continue;
15797                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
15798                                                                                column.getFieldAttr(), true);
15799
15800                                                                columnsInExpr visitor = new columnsInExpr();
15801                                                                column.getExpr().inOrderTraverse(visitor);
15802                                                                List<TObjectName> objectNames = visitor.getObjectNames();
15803                                                                List<TParseTreeNode> functions = visitor.getFunctions();
15804                                                                List<TParseTreeNode> constants = visitor.getConstants();
15805                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15806
15807                                                                if (functions != null && !functions.isEmpty()) {
15808                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
15809                                                                }
15810                                                                if (subquerys != null && !subquerys.isEmpty()) {
15811                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
15812                                                                }
15813                                                                if (objectNames != null && !objectNames.isEmpty()) {
15814                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
15815                                                                }
15816                                                                if (constants != null && !constants.isEmpty()) {
15817                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
15818                                                                                        functions);
15819                                                                }
15820                                                        }
15821                                                }
15822                                        } else {
15823                                                if(table.getTableType() == ETableSource.pivoted_table) {
15824                                                        continue;
15825                                                }
15826                                                if (table.getTableType() == ETableSource.rowList) {
15827                                                        List<TResultColumnList> rowList = table.getValueClause().getRows();
15828                                                        
15829                                                        QueryTable tableModel = modelFactory.createQueryTable(table);
15830                                                        TAliasClause aliasClause = table.getAliasClause();
15831                                                        if (aliasClause != null && aliasClause.getColumns() != null) {
15832                                                                for (TObjectName column : aliasClause.getColumns()) {
15833                                                                        modelFactory.createResultColumn(tableModel, column, true);
15834                                                                }       
15835                                                        } else {        
15836                                                                int columnCount = rowList.get(0).size();
15837                                                                for (int j = 1; j <= columnCount; j++) {
15838                                                                        TObjectName columnName = new TObjectName();
15839                                                                        columnName.setString("column" + j);
15840                                                                        modelFactory.createResultColumn(tableModel, columnName, true);
15841                                                                }
15842                                                        }
15843                                                        tableModel.setDetermined(true);
15844                                                        
15845                                                        for (TResultColumnList resultColumnList : rowList) {
15846                                                                for (int j = 0; j < resultColumnList.size(); j++) {
15847                                                                        TResultColumn resultColumn = resultColumnList.getResultColumn(j);
15848                                                                        analyzeValueColumn(tableModel.getColumns().get(j), resultColumn, EffectType.select);
15849                                                                }
15850                                                        }
15851                                                        
15852                                                } else {
15853                                                        Table tableModel = modelFactory.createTable(table);
15854                                                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
15855                                                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
15856
15857                                                                if (object.getDbObjectType() == EDbObjectType.variable) {
15858                                                                        continue;
15859                                                                }
15860
15861                                                                if (object.getColumnNameOnly().startsWith("@")
15862                                                                                && (option.getVendor() == EDbVendor.dbvmssql
15863                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
15864                                                                        continue;
15865                                                                }
15866
15867                                                                if (object.getColumnNameOnly().startsWith(":")
15868                                                                                && (option.getVendor() == EDbVendor.dbvhana
15869                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
15870                                                                        continue;
15871                                                                }
15872
15873                                                                if (isBuiltInFunctionName(object) && isFromFunction(object)) {
15874                                                                        continue;
15875                                                                }
15876
15877                                                                if (!"*".equals(getColumnName(object))) {
15878                                                                        if (isStructColumn(object)) {
15879
15880                                                                        } else {
15881                                                                                if (pivotedTable != null) {
15882                                                                                        ResultColumn resultColumn = getPivotedTableColumn(pivotedTable, object);
15883                                                                                        if (resultColumn != null) {
15884                                                                                                continue;
15885                                                                                        }
15886                                                                                }
15887                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
15888                                                                                                false);
15889                                                                                if(tableColumn == null) {
15890                                                                                        continue;
15891                                                                                }
15892                                                                                if (table.getUnnestClause() != null
15893                                                                                                && table.getUnnestClause().getArrayExpr() != null) {
15894                                                                                        columnsInExpr visitor = new columnsInExpr();
15895                                                                                        table.getUnnestClause().getArrayExpr().inOrderTraverse(visitor);
15896
15897                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
15898                                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
15899
15900                                                                                        if (functions != null && !functions.isEmpty()) {
15901                                                                                                analyzeFunctionDataFlowRelation(tableColumn, functions,
15902                                                                                                                EffectType.select);
15903
15904                                                                                        }
15905
15906                                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15907                                                                                        if (subquerys != null && !subquerys.isEmpty()) {
15908                                                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys,
15909                                                                                                                EffectType.select);
15910                                                                                        }
15911
15912                                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select,
15913                                                                                                        functions);
15914
15915                                                                                        List<TParseTreeNode> constants = visitor.getConstants();
15916                                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
15917                                                                                                        functions);
15918                                                                                }
15919                                                                        }
15920                                                                } else {
15921                                                                        boolean flag = false;
15922                                                                        for (TObjectName column : getTableLinkedColumns(table)) {
15923                                                                                if ("*".equals(getColumnName(column))) {
15924                                                                                        continue;
15925                                                                                }
15926                                                                                if (column.getLocation() != ESqlClause.where
15927                                                                                                && column.getLocation() != ESqlClause.joinCondition) {
15928                                                                                        flag = true;
15929                                                                                }
15930                                                                        }
15931                                                                        if (!flag) {
15932                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
15933                                                                                                false);
15934                                                                                if (tableColumn != null && !tableModel.hasSQLEnv()) {
15935                                                                                        tableColumn.setShowStar(true);
15936                                                                                }
15937                                                                        }
15938                                                                }
15939
15940                                                        }
15941                                                }
15942                                        }
15943                                }
15944                                else {
15945                                        modelFactory.createTable(table);
15946                                }
15947                        }
15948
15949                        if (pivotedTable!=null) {
15950                                for (TJoin join : stmt.getJoins()) {
15951                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
15952                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
15953                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
15954                                                } else {
15955                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
15956                                                }
15957                                                stmtStack.pop();
15958                                                return;
15959                                        }
15960                                }
15961                        }
15962
15963                        if (!stmt.isCombinedQuery()) {
15964                                Object queryModel = modelManager.getModel(stmt.getResultColumnList());
15965
15966                                if (queryModel == null) {
15967                                        TSelectSqlStatement parentStmt = getParentSetSelectStmt(stmt);
15968                                        if (isTopResultSet(stmt) || parentStmt == null) {
15969                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt,
15970                                                                isTopResultSet(stmt) && isShowTopSelectResultSet() && stmt.getIntoClause() == null);
15971
15972                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
15973
15974                                                boolean isDetermined = true;
15975                                                Map<String, AtomicInteger> keyMap = new HashMap<String, AtomicInteger>();
15976                                                Set<String> columnNames = new HashSet<String>();
15977                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
15978                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
15979
15980                                                        if (column.getExpr().getComparisonType() == EComparisonType.equals
15981                                                                        && column.getExpr().getLeftOperand().getObjectOperand() != null) {
15982                                                                TObjectName columnObject = column.getExpr().getLeftOperand().getObjectOperand();
15983
15984                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
15985                                                                                columnObject);
15986                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
15987                                                                        Table variable = modelManager
15988                                                                                        .getTableByName(DlineageUtil.getTableFullName(columnObject.toString()));
15989                                                                        if (variable != null) {
15990                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15991                                                                                relation.setEffectType(EffectType.select);
15992                                                                                TableColumn columnModel = variable.getColumns().get(0);
15993                                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
15994                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15995                                                                        } else {
15996                                                                                variable = modelFactory.createVariable(columnObject);
15997                                                                                variable.setCreateTable(true);
15998                                                                                variable.setSubType(SubType.record);
15999                                                                                TObjectName variableProperties = new TObjectName();
16000                                                                                variableProperties.setString("*");
16001                                                                                TableColumn variableProperty = modelFactory.createTableColumn(variable,
16002                                                                                                variableProperties, true);
16003                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16004                                                                                relation.setEffectType(EffectType.select);
16005                                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
16006                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16007                                                                        }
16008                                                                }
16009
16010                                                                columnsInExpr visitor = new columnsInExpr();
16011                                                                column.getExpr().getRightOperand().inOrderTraverse(visitor);
16012
16013                                                                List<TObjectName> objectNames = visitor.getObjectNames();
16014                                                                List<TParseTreeNode> functions = visitor.getFunctions();
16015
16016                                                                if (functions != null && !functions.isEmpty()) {
16017                                                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.select);
16018
16019                                                                }
16020
16021                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16022                                                                if (subquerys != null && !subquerys.isEmpty()) {
16023                                                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
16024                                                                }
16025
16026                                                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
16027
16028                                                                List<TParseTreeNode> constants = visitor.getConstants();
16029                                                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
16030                                                        } else {
16031                                                                if (column.getFieldAttr() != null && isStructColumn(column.getFieldAttr())) {
16032                                                                        Table table = modelFactory.createTable(column.getFieldAttr().getSourceTable());
16033                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
16034                                                                                TObjectName columnName = new TObjectName();
16035                                                                                if (table.getColumns().get(j).getName().equals(table.getAlias())) {
16036                                                                                        if (!SQLUtil.isEmpty(column.getColumnAlias())) {
16037                                                                                                columnName.setString(column.getColumnAlias());
16038                                                                                        } else {
16039                                                                                                columnName.setString(table.getColumns().get(j).getName());
16040                                                                                        }
16041                                                                                } else {
16042                                                                                        columnName.setString(
16043                                                                                                        table.getAlias() + "." + table.getColumns().get(j).getName());
16044                                                                                }
16045                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16046                                                                                                columnName);
16047                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16048                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16049                                                                                relationship.addSource(
16050                                                                                                new TableColumnRelationshipElement(table.getColumns().get(j)));
16051                                                                        }
16052                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.struct_t) {
16053                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
16054                                                                        String functionName = getResultSetName(function);
16055                                                                        for (int j = 0; j < function.getColumns().size(); j++) {
16056                                                                                TObjectName columnName = new TObjectName();
16057                                                                                if (column.getAliasClause() != null) {
16058                                                                                        columnName.setString(column.getAliasClause() + "."
16059                                                                                                        + function.getColumns().get(j).getName());
16060                                                                                }
16061                                                                                else {
16062                                                                                        columnName.setString(functionName + "."
16063                                                                                                        + function.getColumns().get(j).getName());
16064                                                                                }
16065                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16066                                                                                                columnName);
16067                                                                                resultColumn.setStruct(true);
16068                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16069                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16070                                                                                relationship.addSource(
16071                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(j)));
16072                                                                        }
16073                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.array_t) {
16074                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
16075                                                                        String functionName = getResultSetName(function);
16076                                                                        if (function.getColumns().size() == 1) {
16077                                                                                // Scalar result inside ARRAY() - use just alias as column name
16078                                                                                TObjectName columnName = new TObjectName();
16079                                                                                if (column.getAliasClause() != null) {
16080                                                                                        columnName.setString(column.getAliasClause().toString());
16081                                                                                } else {
16082                                                                                        columnName.setString(functionName);
16083                                                                                }
16084                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16085                                                                                                columnName);
16086                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16087                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16088                                                                                relationship.addSource(
16089                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(0)));
16090                                                                        } else {
16091                                                                                for (int j = 0; j < function.getColumns().size(); j++) {
16092                                                                                        TObjectName columnName = new TObjectName();
16093                                                                                        if (column.getAliasClause() != null) {
16094                                                                                                columnName.setString(column.getAliasClause() + "."
16095                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
16096                                                                                        }
16097                                                                                        else {
16098                                                                                                columnName.setString(functionName + "."
16099                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
16100                                                                                        }
16101                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16102                                                                                                        columnName);
16103                                                                                        resultColumn.setStruct(true);
16104                                                                                        DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16105                                                                                        relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16106                                                                                        relationship.addSource(
16107                                                                                                        new ResultColumnRelationshipElement(function.getColumns().get(j)));
16108                                                                                }
16109                                                                        }
16110                                                                } else if (column.getExpr().getExprList() != null && column.getExpr().getExpressionType() == EExpressionType.array_t) {
16111                                                                        if(column.getExpr().getObjectOperand()!=null) {
16112                                                                                TObjectName exprColumnName = column.getExpr().getObjectOperand();
16113                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
16114                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
16115                                                                                        TableColumn tableColumn = table.getColumns().get(z);
16116                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
16117                                                                                                TObjectName columnName = new TObjectName();
16118                                                                                                if (column.getAliasClause() != null) {
16119                                                                                                        columnName.setString(column.getAliasClause().toString());
16120                                                                                                } else {
16121                                                                                                        columnName
16122                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
16123                                                                                                }
16124                                                                                                ResultColumn resultColumn = modelFactory
16125                                                                                                                .createResultColumn(resultSetModel, columnName);
16126                                                                                                resultColumn.setStruct(true);
16127                                                                                                DataFlowRelationship relationship = modelFactory
16128                                                                                                                .createDataFlowRelation();
16129                                                                                                relationship.setTarget(
16130                                                                                                                new ResultColumnRelationshipElement(resultColumn));
16131                                                                                                relationship.addSource(new TableColumnRelationshipElement(
16132                                                                                                                tableColumn));
16133                                                                                        }
16134                                                                                }
16135                                                                        }
16136                                                                        else {
16137                                                                                for (int j = 0; j < column.getExpr().getExprList().size(); j++) {
16138                                                                                        TExpression expression = column.getExpr().getExprList().getExpression(j);
16139                                                                                        if(expression.getExpressionType() == EExpressionType.function_t) {
16140                                                                                                Function function = (Function)createFunction(expression.getFunctionCall());
16141                                                                                                String functionName = getResultSetName(function);
16142                                                                                                if (function != null && function.getColumns() != null) {
16143                                                                                                        if (function.getColumns().size() == 1) {
16144                                                                                                                // Scalar function inside ARRAY[] (e.g., ARRAY[TO_JSON_STRING(col)])
16145                                                                                                                // Use just the alias as column name, not alias.functionName
16146                                                                                                                TObjectName columnName = new TObjectName();
16147                                                                                                                if (column.getAliasClause() != null) {
16148                                                                                                                        columnName.setString(column.getAliasClause().toString());
16149                                                                                                                } else {
16150                                                                                                                        columnName.setString(functionName);
16151                                                                                                                }
16152                                                                                                                ResultColumn resultColumn = modelFactory
16153                                                                                                                                .createResultColumn(resultSetModel, columnName);
16154                                                                                                                DataFlowRelationship relationship = modelFactory
16155                                                                                                                                .createDataFlowRelation();
16156                                                                                                                relationship.setTarget(
16157                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
16158                                                                                                                relationship.addSource(new ResultColumnRelationshipElement(
16159                                                                                                                                function.getColumns().get(0)));
16160                                                                                                        } else {
16161                                                                                                                // Multi-column (struct-like) function - use dotted naming
16162                                                                                                                for (int x = 0; x < function.getColumns().size(); x++) {
16163                                                                                                                        TObjectName columnName = new TObjectName();
16164                                                                                                                        if (column.getAliasClause() != null) {
16165                                                                                                                                columnName.setString(column.getAliasClause() + "."
16166                                                                                                                                                + getColumnNameOnly(
16167                                                                                                                                                function.getColumns().get(x).getName()));
16168                                                                                                                        } else {
16169                                                                                                                                columnName.setString(
16170                                                                                                                                                functionName + "." + getColumnNameOnly(
16171                                                                                                                                                                function.getColumns().get(x).getName()));
16172                                                                                                                        }
16173                                                                                                                        ResultColumn resultColumn = modelFactory
16174                                                                                                                                        .createResultColumn(resultSetModel, columnName);
16175                                                                                                                        resultColumn.setStruct(true);
16176                                                                                                                        DataFlowRelationship relationship = modelFactory
16177                                                                                                                                        .createDataFlowRelation();
16178                                                                                                                        relationship.setTarget(
16179                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
16180                                                                                                                        relationship.addSource(new ResultColumnRelationshipElement(
16181                                                                                                                                        function.getColumns().get(x)));
16182                                                                                                                }
16183                                                                                                        }
16184                                                                                                }
16185                                                                                        }
16186                                                                                        else if(expression.getExpressionType() == EExpressionType.simple_object_name_t) {
16187                                                                                                TObjectName exprColumnName = expression.getObjectOperand();
16188                                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
16189                                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
16190                                                                                                        TableColumn tableColumn = table.getColumns().get(z);
16191                                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
16192                                                                                                                TObjectName columnName = new TObjectName();
16193                                                                                                                if (column.getAliasClause() != null) {
16194                                                                                                                        columnName.setString(column.getAliasClause().toString());
16195                                                                                                                } else {
16196                                                                                                                        columnName
16197                                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
16198                                                                                                                }
16199                                                                                                                ResultColumn resultColumn = modelFactory
16200                                                                                                                                .createResultColumn(resultSetModel, columnName);
16201                                                                                                                resultColumn.setStruct(true);
16202                                                                                                                DataFlowRelationship relationship = modelFactory
16203                                                                                                                                .createDataFlowRelation();
16204                                                                                                                relationship.setTarget(
16205                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
16206                                                                                                                relationship.addSource(new TableColumnRelationshipElement(
16207                                                                                                                                tableColumn));
16208                                                                                                        }
16209                                                                                                }
16210                                                                                        }
16211                                                                                }
16212                                                                        }
16213                                                                } else {
16214                                                                        if ("*".equals(column.getColumnNameOnly())) {
16215                                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
16216                                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
16217                                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
16218                                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
16219                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
16220                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
16221                                                                                        }
16222                                                                                }
16223                                                                                
16224                                                                                TObjectName columnObject = column.getFieldAttr();
16225                                                                                List<TTable> sourceTables = columnObject.getSourceTableList();
16226                                                                                if (sourceTables != null && !sourceTables.isEmpty()) {
16227                                                                                        boolean[] determine = new boolean[sourceTables.size()];
16228                                                                                        for (int k = 0; k < sourceTables.size(); k++) {
16229                                                                                                TTable sourceTable = sourceTables.get(k);
16230                                                                                                Object tableModel = modelManager.getModel(sourceTable);
16231                                                                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
16232                                                                                                        Table table = (Table) tableModel;
16233                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
16234                                                                                                                TableColumn tableColumn = table.getColumns().get(j);
16235                                                                                                                if (column.getExceptColumnList() != null) {
16236                                                                                                                        boolean except = false;
16237                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
16238                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
16239                                                                                                                                        except = true;
16240                                                                                                                                        break;
16241                                                                                                                                }
16242                                                                                                                        }
16243                                                                                                                        if (!except && tableColumn.isStruct()) {
16244                                                                                                                                List<String> names = SQLUtil
16245                                                                                                                                                .parseNames(tableColumn.getName());
16246                                                                                                                                for (String name : names) {
16247                                                                                                                                        for (TObjectName objectName : column
16248                                                                                                                                                        .getExceptColumnList()) {
16249                                                                                                                                                if (getColumnName(objectName.toString())
16250                                                                                                                                                                .equals(getColumnName(name))) {
16251                                                                                                                                                        except = true;
16252                                                                                                                                                        break;
16253                                                                                                                                                }
16254                                                                                                                                        }
16255                                                                                                                                        if (except) {
16256                                                                                                                                                break;
16257                                                                                                                                        }
16258                                                                                                                                }
16259                                                                                                                        }
16260                                                                                                                        if(except){
16261                                                                                                                                continue;
16262                                                                                                                        }
16263                                                                                                                }
16264                                                                                                                
16265                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16266                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16267                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16268                                                                                                                        Transform transform = new Transform();
16269                                                                                                                        transform.setType(Transform.EXPRESSION);
16270                                                                                                                        TObjectName expression = new TObjectName();
16271                                                                                                                        expression.setString(expr.first);
16272                                                                                                                transform.setCode(expression);
16273                                                                                                                        resultColumn.setTransform(transform);
16274                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16275                                                                                                                }
16276                                                                                                                else {
16277                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
16278                                                                                                                        //bigquery
16279                                                                                                                        boolean exist = false;
16280                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
16281                                                                                                                                if (!keyMap.containsKey(columnName)) {
16282                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
16283                                                                                                                                } else {
16284                                                                                                                                        while (columnNames.contains(columnName)) {
16285                                                                                                                                                if(columnName.indexOf("*") == -1 && stmt.toString().matches("(?is).*using\\s*\\(\\s*"+columnName+"\\s*\\).*")) {
16286                                                                                                                                                        exist = true;
16287                                                                                                                                                        break;
16288                                                                                                                                                } else if (keyMap.containsKey(columnName)) {
16289                                                                                                                                                        int index = keyMap.get(columnName)
16290                                                                                                                                                                        .incrementAndGet();
16291                                                                                                                                                        columnName = columnName + index;
16292                                                                                                                                                }
16293                                                                                                                                        }
16294                                                                                                                                }
16295                                                                                                                                columnNames.add(columnName);
16296                                                                                                                        }
16297                                                                                                                        if (exist) {
16298                                                                                                                                String targetColumn = columnName;
16299                                                                                                                                DataFlowRelationship relation = modelFactory
16300                                                                                                                                                .createDataFlowRelation();
16301                                                                                                                                relation.setEffectType(EffectType.select);
16302                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(
16303                                                                                                                                                resultSetModel.getColumns().stream()
16304                                                                                                                                                                .filter(t -> t.getName()
16305                                                                                                                                                                                .equalsIgnoreCase(targetColumn))
16306                                                                                                                                                                .findFirst().get()));
16307                                                                                                                                relation.addSource(new TableColumnRelationshipElement(
16308                                                                                                                                                tableColumn));
16309                                                                                                                                continue;
16310                                                                                                                        }
16311                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
16312                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
16313                                                                                                                                columnName = tableColumn.getName();
16314                                                                                                                        }
16315                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
16316                                                                                                                        resultColumn.setStruct(tableColumn.isStruct());
16317                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16318                                                                                                                        relation.setEffectType(EffectType.select);
16319                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16320                                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
16321                                                                                                                }
16322                                                                                                        }
16323                                                                                                        determine[k] = true;
16324                                                                                                        continue;
16325                                                                                                }
16326                                                                                                else if (tableModel instanceof ResultSet && ((ResultSet) tableModel).isDetermined()) {
16327                                                                                                        ResultSet table = (ResultSet) tableModel;
16328                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
16329                                                                                                                ResultColumn tableColumn = table.getColumns().get(j);
16330                                                                                                                if (column.getExceptColumnList() != null) {
16331                                                                                                                        boolean except = false;
16332                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
16333                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
16334                                                                                                                                        except = true;
16335                                                                                                                                        break;
16336                                                                                                                                }
16337                                                                                                                        }
16338                                                                                                                        if (!except && tableColumn.isStruct()) {
16339                                                                                                                                List<String> names = SQLUtil
16340                                                                                                                                                .parseNames(tableColumn.getName());
16341                                                                                                                                for (String name : names) {
16342                                                                                                                                        for (TObjectName objectName : column
16343                                                                                                                                                        .getExceptColumnList()) {
16344                                                                                                                                                if (getColumnName(objectName.toString())
16345                                                                                                                                                                .equals(getColumnName(name))) {
16346                                                                                                                                                        except = true;
16347                                                                                                                                                        break;
16348                                                                                                                                                }
16349                                                                                                                                        }
16350                                                                                                                                        if (except) {
16351                                                                                                                                                break;
16352                                                                                                                                        }
16353                                                                                                                                }
16354                                                                                                                        }
16355                                                                                                                        if(except){
16356                                                                                                                                continue;
16357                                                                                                                        }
16358                                                                                                                }
16359                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16360                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16361                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16362                                                                                                                        Transform transform = new Transform();
16363                                                                                                                        transform.setType(Transform.EXPRESSION);
16364                                                                                                                        TObjectName expression = new TObjectName();
16365                                                                                                                        expression.setString(expr.first);
16366                                                                                                                transform.setCode(expression);
16367                                                                                                                        resultColumn.setTransform(transform);
16368                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16369                                                                                                                }
16370                                                                                                                else if(tableColumn.getRefColumnName()!=null) {
16371                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, tableColumn.getRefColumnName());
16372                                                                                                                        if(tableColumn.isStruct()) {
16373                                                                                                                                resultColumn.setStruct(true);
16374                                                                                                                        }
16375                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16376                                                                                                                        relation.setEffectType(EffectType.select);
16377                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16378                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16379                                                                                                                }
16380                                                                                                                else {
16381                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
16382                                                                                                                        //bigquery
16383                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
16384                                                                                                                                if (!keyMap.containsKey(columnName)) {
16385                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
16386                                                                                                                                } else {
16387                                                                                                                                        while (columnNames.contains(columnName)) {
16388                                                                                                                                                int index = keyMap.get(columnName)
16389                                                                                                                                                                .incrementAndGet();
16390                                                                                                                                                columnName = columnName + index;
16391                                                                                                                                        }
16392                                                                                                                                }
16393                                                                                                                                columnNames.add(columnName);
16394                                                                                                                        }
16395                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
16396                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
16397                                                                                                                                columnName = tableColumn.getName();
16398                                                                                                                        }
16399                                                                                                                        if (modelManager.getModel(column) instanceof ResultColumn) {
16400                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16401                                                                                                                                relation.setEffectType(EffectType.select);
16402                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement((ResultColumn)modelManager.getModel(column)));
16403                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16404                                                                                                                        }
16405                                                                                                                        else {
16406                                                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
16407                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16408                                                                                                                                relation.setEffectType(EffectType.select);
16409                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16410                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16411                                                                                                                        }
16412                                                                                                                }
16413                                                                                                        }
16414                                                                                                        determine[k] = true;
16415                                                                                                        continue;
16416                                                                                                }
16417                                                                                                else {
16418                                                                                                        ResultColumn resultColumn = modelFactory
16419                                                                                                                        .createResultColumn(resultSetModel, column);
16420                                                                                                        if (tableModel instanceof Function) {
16421                                                                                                
16422                                                                                                        } else {
16423                                                                                                                TObjectName[] columns = modelManager
16424                                                                                                                                .getTableColumns(sourceTable);
16425                                                                                                                for (int j = 0; j < columns.length; j++) {
16426                                                                                                                        TObjectName columnName = columns[j];
16427                                                                                                                        if (columnName == null
16428                                                                                                                                        || "*".equals(getColumnName(columnName))) {
16429                                                                                                                                continue;
16430                                                                                                                        }
16431                                                                                                                        if (isStructColumn(columnName)) {
16432                                                                                                                                continue;
16433                                                                                                                        }
16434
16435                                                                                                                        resultColumn.bindStarLinkColumn(columnName);
16436                                                                                                                        if (column.getExceptColumnList() != null) {
16437                                                                                                                                for (TObjectName objectName : column
16438                                                                                                                                                .getExceptColumnList()) {
16439                                                                                                                                        resultColumn.unbindStarLinkColumn(objectName);
16440                                                                                                                                }
16441                                                                                                                        }
16442                                                                                                                }
16443                                                                                                                if (tableModel instanceof ResultSet) {
16444                                                                                                                        ResultSet queryTable = (ResultSet) tableModel;
16445                                                                                                                        if (!containStarColumn(queryTable)) {
16446                                                                                                                                resultColumn.setShowStar(false);
16447                                                                                                                        }
16448                                                                                                                }
16449                                                                                                                if (tableModel instanceof Table) {
16450                                                                                                                        Table table = (Table) tableModel;
16451                                                                                                                        if (table.isCreateTable()) {
16452                                                                                                                                resultColumn.setShowStar(false);
16453                                                                                                                        }
16454                                                                                                                }
16455                                                                                                        }
16456                                                                                                }
16457                                                                                        }
16458                                                                                        if(!Arrays.toString(determine).contains("false")) {
16459                                                                                                continue;
16460                                                                                        }
16461                                                                                } else {
16462                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
16463                                                                                        TTableList tables = stmt.getTables();
16464                                                                                        for (int k = 0; k < tables.size(); k++) {
16465                                                                                                TTable table = tables.getTable(k);
16466                                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
16467                                                                                                for (int j = 0; j < columns.length; j++) {
16468                                                                                                        TObjectName columnName = columns[j];
16469                                                                                                        if (columnName == null) {
16470                                                                                                                continue;
16471                                                                                                        }
16472                                                                                                        if ("*".equals(getColumnName(columnName))) {
16473                                                                                                                if (modelManager.getModel(table) instanceof Table) {
16474                                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
16475                                                                                                                        if (tableModel != null
16476                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
16477                                                                                                                                for (TableColumn item : tableModel.getColumns()) {
16478                                                                                                                                        resultColumn
16479                                                                                                                                                        .bindStarLinkColumn(item.getColumnObject());
16480                                                                                                                                        if (table.getSubquery() == null
16481                                                                                                                                                        && table.getCTE() == null
16482                                                                                                                                                        && !tableModel.isCreateTable()) {
16483                                                                                                                                                resultColumn.setShowStar(true);
16484                                                                                                                                        }
16485                                                                                                                                }
16486                                                                                                                        }
16487                                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
16488                                                                                                                        QueryTable tableModel = (QueryTable) modelManager
16489                                                                                                                                        .getModel(table);
16490                                                                                                                        if (tableModel != null
16491                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
16492                                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
16493                                                                                                                                        if (item.hasStarLinkColumn()) {
16494                                                                                                                                                for (TObjectName starLinkColumn : item
16495                                                                                                                                                                .getStarLinkColumnList()) {
16496                                                                                                                                                        resultColumn
16497                                                                                                                                                                        .bindStarLinkColumn(starLinkColumn);
16498                                                                                                                                                }
16499                                                                                                                                        } else if (item
16500                                                                                                                                                        .getColumnObject() instanceof TObjectName) {
16501                                                                                                                                                resultColumn.bindStarLinkColumn(
16502                                                                                                                                                                (TObjectName) item.getColumnObject());
16503                                                                                                                                        } else if (item
16504                                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
16505                                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
16506                                                                                                                                                                .getColumnObject();
16507                                                                                                                                                TObjectName tableColumnObject = queryTableColumn
16508                                                                                                                                                                .getFieldAttr();
16509                                                                                                                                                if (tableColumnObject != null) {
16510                                                                                                                                                        resultColumn.bindStarLinkColumn(
16511                                                                                                                                                                        tableColumnObject);
16512                                                                                                                                                } else if (queryTableColumn
16513                                                                                                                                                                .getAliasClause() != null) {
16514                                                                                                                                                        resultColumn.bindStarLinkColumn(
16515                                                                                                                                                                        queryTableColumn.getAliasClause()
16516                                                                                                                                                                                        .getAliasName());
16517                                                                                                                                                }
16518                                                                                                                                        }
16519                                                                                                                                }
16520                                                                                                                        }
16521                                                                                                                }
16522                                                                                                                continue;
16523                                                                                                        }
16524                                                                                                        resultColumn.bindStarLinkColumn(columnName);
16525                                                                                                }
16526                                                                                        }
16527                                                                                }
16528                                                                                isDetermined = false;
16529                                                                        }
16530                                                                        else {
16531                                                                                if(column.getAliasClause()!=null && column.getAliasClause().getColumns()!=null) {
16532                                                                                        for(TObjectName aliasColumn: column.getAliasClause().getColumns()) {
16533                                                                                                modelFactory.createResultColumn(resultSetModel, aliasColumn);
16534                                                                                        }
16535                                                                                }
16536                                                                                else {
16537                                                                                        modelFactory.createResultColumn(resultSetModel, column);
16538                                                                                }
16539                                                                        }
16540                                                                        analyzeResultColumn(column, EffectType.select);
16541                                                                }
16542                                                        }
16543                                                }
16544                                                if (isDetermined) {
16545                                                        resultSetModel.setDetermined(isDetermined);
16546                                                }
16547                                        }
16548
16549                                        TSelectSqlStatement parent = getParentSetSelectStmt(stmt);
16550                                        if (parent != null && parent.getSetOperatorType() != ESetOperatorType.none) {
16551                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt, false);
16552                                                if(queryModel == null) {
16553                                                        queryModel = resultSetModel;
16554                                                } 
16555
16556                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
16557
16558                                                boolean isDetermined = true;
16559                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
16560                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
16561                                                        if ("*".equals(column.getColumnNameOnly())) {
16562                                                                
16563                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
16564                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
16565                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
16566                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
16567                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
16568                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
16569                                                                        }
16570                                                                }
16571                                                                
16572                                                                TObjectName columnObject = column.getFieldAttr();
16573                                                                TTable sourceTable = columnObject.getSourceTable();
16574                                                                if (sourceTable != null) {
16575                                                                        Object tableModel = modelManager.getModel(sourceTable);
16576                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
16577                                                                                Table table = (Table) tableModel;
16578                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
16579                                                                                        TableColumn tableColumn = table.getColumns().get(j);
16580                                                                                        if (column.getExceptColumnList() != null) {
16581                                                                                                boolean except = false;
16582                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
16583                                                                                                        if (getColumnName(objectName.toString())
16584                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
16585                                                                                                                except = true;
16586                                                                                                                break;
16587                                                                                                        }
16588                                                                                                }
16589                                                                                                if (!except && tableColumn.isStruct()) {
16590                                                                                                        List<String> names = SQLUtil
16591                                                                                                                        .parseNames(tableColumn.getName());
16592                                                                                                        for (String name : names) {
16593                                                                                                                for (TObjectName objectName : column
16594                                                                                                                                .getExceptColumnList()) {
16595                                                                                                                        if (getColumnName(objectName.toString())
16596                                                                                                                                        .equals(getColumnName(name))) {
16597                                                                                                                                except = true;
16598                                                                                                                                break;
16599                                                                                                                        }
16600                                                                                                                }
16601                                                                                                                if (except) {
16602                                                                                                                        break;
16603                                                                                                                }
16604                                                                                                        }
16605                                                                                                }
16606                                                                                                if (except) {
16607                                                                                                        continue;
16608                                                                                                }
16609                                                                                        }
16610                                                                                        
16611                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16612                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16613                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16614                                                                                                Transform transform = new Transform();
16615                                                                                                transform.setType(Transform.EXPRESSION);
16616                                                                                                TObjectName expression = new TObjectName();
16617                                                                                                expression.setString(expr.first);
16618                                                                                        transform.setCode(expression);
16619                                                                                                resultColumn.setTransform(transform);
16620                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16621                                                                                        }
16622                                                                                        else {
16623                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16624                                                                                                                resultSetModel, column, tableColumn.getName());
16625                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16626                                                                                                relation.setEffectType(EffectType.select);
16627                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16628                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
16629                                                                                        }
16630                                                                                }
16631                                                                                continue;
16632                                                                        } else if (tableModel instanceof ResultSet
16633                                                                                        && ((ResultSet) tableModel).isDetermined()) {
16634                                                                                ResultSet table = (ResultSet) tableModel;
16635                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
16636                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
16637                                                                                        if (column.getExceptColumnList() != null) {
16638                                                                                                boolean except = false;
16639                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
16640                                                                                                        if (getColumnName(objectName.toString())
16641                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
16642                                                                                                                except = true;
16643                                                                                                                break;
16644                                                                                                        }
16645                                                                                                }
16646                                                                                                if (!except && tableColumn.isStruct()) {
16647                                                                                                        List<String> names = SQLUtil
16648                                                                                                                        .parseNames(tableColumn.getName());
16649                                                                                                        for (String name : names) {
16650                                                                                                                for (TObjectName objectName : column
16651                                                                                                                                .getExceptColumnList()) {
16652                                                                                                                        if (getColumnName(objectName.toString())
16653                                                                                                                                        .equals(getColumnName(name))) {
16654                                                                                                                                except = true;
16655                                                                                                                                break;
16656                                                                                                                        }
16657                                                                                                                }
16658                                                                                                                if (except) {
16659                                                                                                                        break;
16660                                                                                                                }
16661                                                                                                        }
16662                                                                                                }
16663                                                                                                if (except) {
16664                                                                                                        continue;
16665                                                                                                }
16666                                                                                        }
16667                                                                                        
16668                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16669                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16670                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16671                                                                                                Transform transform = new Transform();
16672                                                                                                transform.setType(Transform.EXPRESSION);
16673                                                                                                TObjectName expression = new TObjectName();
16674                                                                                                expression.setString(expr.first);
16675                                                                                        transform.setCode(expression);
16676                                                                                                resultColumn.setTransform(transform);
16677                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16678                                                                                        }
16679                                                                                        else if (tableColumn.getRefColumnName() != null) {
16680                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16681                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
16682                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16683                                                                                                relation.setEffectType(EffectType.select);
16684                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16685                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16686                                                                                        } else {
16687                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16688                                                                                                                resultSetModel, column, tableColumn.getName());
16689                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16690                                                                                                relation.setEffectType(EffectType.select);
16691                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16692                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16693                                                                                        }
16694                                                                                }
16695                                                                                continue;
16696                                                                        }
16697                                                                        else {
16698                                                                                isDetermined = false;
16699                                                                        }
16700                                                                }
16701                                                                
16702                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
16703                                                                
16704                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
16705                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
16706                                                                        for (int j = 0; j < columns.length; j++) {
16707                                                                                TObjectName columnName = columns[j];
16708                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
16709                                                                                        continue;
16710                                                                                }
16711                                                                                resultColumn.bindStarLinkColumn(columnName);
16712                                                                        }
16713
16714                                                                        if (modelManager.getModel(sourceTable) instanceof Table) {
16715                                                                                Table tableModel = (Table) modelManager.getModel(sourceTable);
16716                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16717                                                                                        for (TableColumn item : tableModel.getColumns()) {
16718                                                                                                if ("*".equals(getColumnName(item.getColumnObject()))) {
16719                                                                                                        continue;
16720                                                                                                }
16721                                                                                                resultColumn.bindStarLinkColumn(item.getColumnObject());
16722                                                                                        }
16723                                                                                }
16724                                                                        } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
16725                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
16726                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16727                                                                                        for (ResultColumn item : tableModel.getColumns()) {
16728                                                                                                if (item.hasStarLinkColumn()) {
16729                                                                                                        for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
16730                                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
16731                                                                                                                        continue;
16732                                                                                                                }
16733                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
16734                                                                                                        }
16735                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
16736                                                                                                        TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
16737                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
16738                                                                                                                continue;
16739                                                                                                        }
16740                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
16741                                                                                                }
16742                                                                                        }
16743                                                                                }
16744                                                                        }
16745
16746                                                                } else {
16747                                                                        TTableList tables = stmt.getTables();
16748                                                                        for (int k = 0; k < tables.size(); k++) {
16749                                                                                TTable table = tables.getTable(k);
16750                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
16751                                                                                for (int j = 0; j < columns.length; j++) {
16752                                                                                        TObjectName columnName = columns[j];
16753                                                                                        if (columnName == null) {
16754                                                                                                continue;
16755                                                                                        }
16756                                                                                        if ("*".equals(getColumnName(columnName))) {
16757                                                                                                if (modelManager.getModel(table) instanceof Table) {
16758                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
16759                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16760                                                                                                                for (TableColumn item : tableModel.getColumns()) {
16761                                                                                                                        resultColumn.bindStarLinkColumn(item.getColumnObject());
16762                                                                                                                }
16763                                                                                                        }
16764                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
16765                                                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
16766                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16767                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
16768                                                                                                                        if (item.hasStarLinkColumn()) {
16769                                                                                                                                for (TObjectName starLinkColumn : item
16770                                                                                                                                                .getStarLinkColumnList()) {
16771                                                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
16772                                                                                                                                }
16773                                                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
16774                                                                                                                                resultColumn.bindStarLinkColumn(
16775                                                                                                                                                (TObjectName) item.getColumnObject());
16776                                                                                                                        } else if (item
16777                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
16778                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
16779                                                                                                                                                .getColumnObject();
16780                                                                                                                                TObjectName tableColumnObject = queryTableColumn
16781                                                                                                                                                .getFieldAttr();
16782                                                                                                                                if (tableColumnObject != null) {
16783                                                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
16784                                                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
16785                                                                                                                                        resultColumn.bindStarLinkColumn(queryTableColumn
16786                                                                                                                                                        .getAliasClause().getAliasName());
16787                                                                                                                                }
16788                                                                                                                        }
16789                                                                                                                }
16790                                                                                                        }
16791                                                                                                }
16792
16793                                                                                                continue;
16794                                                                                        }
16795                                                                                        resultColumn.bindStarLinkColumn(columnName);
16796                                                                                }
16797                                                                        }
16798                                                                }
16799                                                        }
16800                                                        else {
16801                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
16802                                                        }
16803                                                        analyzeResultColumn(column, EffectType.select);
16804
16805                                                }
16806                                                
16807                                                resultSetModel.setDetermined(isDetermined);
16808                                        }
16809                                } else {
16810                                        for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
16811                                                TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
16812
16813                                                if (!(queryModel instanceof ResultSet)) {
16814                                                        continue;
16815                                                }
16816                                                
16817                                                ResultSet resultSetModel = (ResultSet)queryModel;
16818                                                        
16819                                                if ("*".equals(column.getColumnNameOnly())) {
16820                                                        TObjectName columnObject = column.getFieldAttr();
16821                                                        TTable sourceTable = columnObject.getSourceTable();
16822                                                        if (column.toString().indexOf(".") == -1 && stmt.getTables().size() > 1) {
16823                                                                sourceTable = null;
16824                                                        }
16825                                                        if (sourceTable != null) {
16826                                                                {
16827                                                                        Object tableModel = modelManager.getModel(sourceTable);
16828                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
16829                                                                                Table table = (Table) tableModel;
16830                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
16831                                                                                        TableColumn tableColumn = table.getColumns().get(j);
16832                                                                                        if (column.getExceptColumnList() != null) {
16833                                                                                                boolean except = false;
16834                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
16835                                                                                                        if (getColumnName(objectName.toString())
16836                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
16837                                                                                                                except = true;
16838                                                                                                                break;
16839                                                                                                        }
16840                                                                                                }
16841                                                                                                if (!except && tableColumn.isStruct()) {
16842                                                                                                        List<String> names = SQLUtil
16843                                                                                                                        .parseNames(tableColumn.getName());
16844                                                                                                        for (String name : names) {
16845                                                                                                                for (TObjectName objectName : column
16846                                                                                                                                .getExceptColumnList()) {
16847                                                                                                                        if (getColumnName(objectName.toString())
16848                                                                                                                                        .equals(getColumnName(name))) {
16849                                                                                                                                except = true;
16850                                                                                                                                break;
16851                                                                                                                        }
16852                                                                                                                }
16853                                                                                                                if (except) {
16854                                                                                                                        break;
16855                                                                                                                }
16856                                                                                                        }
16857                                                                                                }
16858                                                                                                if (except) {
16859                                                                                                        continue;
16860                                                                                                }
16861                                                                                        }
16862                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(
16863                                                                                                        resultSetModel, column, tableColumn.getName());
16864                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16865                                                                                        relation.setEffectType(EffectType.select);
16866                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16867                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
16868                                                                                }
16869                                                                                if(stmt.getResultColumnList().size() == 1) {
16870                                                                                        resultSetModel.setDetermined(true);
16871                                                                                }
16872                                                                                else {
16873                                                                                        int starCount = 0;
16874                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
16875                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
16876                                                                                                                .endsWith("*")) {
16877                                                                                                        starCount += 1;
16878                                                                                                }
16879                                                                                        }
16880                                                                                        if (starCount <= 1) {
16881                                                                                                resultSetModel.setDetermined(true);
16882                                                                                        }
16883                                                                                }
16884                                                                                continue;
16885                                                                        } else if (tableModel instanceof ResultSet
16886                                                                                        && ((ResultSet) tableModel).isDetermined()) {
16887                                                                                ResultSet table = (ResultSet) tableModel;
16888                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
16889                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
16890                                                                                        if (column.getExceptColumnList() != null) {
16891                                                                                                boolean except = false;
16892                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
16893                                                                                                        if (getColumnName(objectName.toString())
16894                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
16895                                                                                                                except = true;
16896                                                                                                                break;
16897                                                                                                        }
16898                                                                                                }
16899                                                                                                if (!except && tableColumn.isStruct()) {
16900                                                                                                        List<String> names = SQLUtil
16901                                                                                                                        .parseNames(tableColumn.getName());
16902                                                                                                        for (String name : names) {
16903                                                                                                                for (TObjectName objectName : column
16904                                                                                                                                .getExceptColumnList()) {
16905                                                                                                                        if (getColumnName(objectName.toString())
16906                                                                                                                                        .equals(getColumnName(name))) {
16907                                                                                                                                except = true;
16908                                                                                                                                break;
16909                                                                                                                        }
16910                                                                                                                }
16911                                                                                                                if (except) {
16912                                                                                                                        break;
16913                                                                                                                }
16914                                                                                                        }
16915                                                                                                }
16916                                                                                                if (except) {
16917                                                                                                        continue;
16918                                                                                                }
16919                                                                                        }
16920                                                                                        if (tableColumn.getRefColumnName() != null) {
16921                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16922                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
16923                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16924                                                                                                relation.setEffectType(EffectType.select);
16925                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16926                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16927                                                                                        } else {
16928                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16929                                                                                                                resultSetModel, column, tableColumn.getName());
16930                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16931                                                                                                relation.setEffectType(EffectType.select);
16932                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16933                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16934                                                                                        }
16935                                                                                }
16936                                                                                if(stmt.getResultColumnList().size() == 1) {
16937                                                                                        resultSetModel.setDetermined(true);
16938                                                                                }
16939                                                                                else {
16940                                                                                        int starCount = 0;
16941                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
16942                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
16943                                                                                                                .endsWith("*")) {
16944                                                                                                        starCount += 1;
16945                                                                                                }
16946                                                                                        }
16947                                                                                        if (starCount <= 1) {
16948                                                                                                resultSetModel.setDetermined(true);
16949                                                                                        }
16950                                                                                }
16951                                                                                continue;
16952                                                                        }
16953                                                                }
16954                                                                
16955                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
16956                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
16957                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
16958                                                                        if (tableModel != null) {
16959                                                                                modelFactory.createTableColumn(tableModel, columnObject, false);
16960                                                                        }
16961                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
16962                                                                        for (int j = 0; j < columns.length; j++) {
16963                                                                                TObjectName columnName = columns[j];
16964                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
16965                                                                                        continue;
16966                                                                                }
16967                                                                                resultColumn.bindStarLinkColumn(columnName);
16968                                                                        }
16969
16970                                                                        if (tableModel.getColumns() != null) {
16971                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
16972                                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
16973                                                                                        TObjectName columnName = tableColumn.getColumnObject();
16974                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
16975                                                                                                continue;
16976                                                                                        }
16977                                                                                        resultColumn.bindStarLinkColumn(columnName);
16978                                                                                }
16979                                                                        }
16980                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
16981                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
16982                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16983                                                                                for (ResultColumn item : tableModel.getColumns()) {
16984                                                                                        if (item.hasStarLinkColumn()) {
16985                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
16986                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
16987                                                                                                }
16988                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
16989                                                                                                resultColumn.bindStarLinkColumn((TObjectName) item.getColumnObject());
16990                                                                                        } else if (item.getColumnObject() instanceof TResultColumn) {
16991                                                                                                TResultColumn queryTableColumn = (TResultColumn) item.getColumnObject();
16992                                                                                                TObjectName tableColumnObject = queryTableColumn.getFieldAttr();
16993                                                                                                if (tableColumnObject != null) {
16994                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
16995                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
16996                                                                                                        resultColumn.bindStarLinkColumn(
16997                                                                                                                        queryTableColumn.getAliasClause().getAliasName());
16998                                                                                                }
16999                                                                                        }
17000                                                                                }
17001                                                                        }
17002                                                                }
17003                                                        } else {
17004                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
17005                                                                TTableList tables = stmt.getTables();
17006                                                                for (int k = 0; k < tables.size(); k++) {
17007                                                                        TTable table = tables.getTable(k);
17008                                                                        TObjectName[] columns = modelManager.getTableColumns(table);
17009                                                                        for (int j = 0; j < columns.length; j++) {
17010                                                                                TObjectName columnName = columns[j];
17011                                                                                if (columnName == null) {
17012                                                                                        continue;
17013                                                                                }
17014                                                                                if ("*".equals(getColumnName(columnName))) {
17015                                                                                        if (modelManager.getModel(table) instanceof Table) {
17016                                                                                                Table tableModel = (Table) modelManager.getModel(table);
17017                                                                                                if (tableModel != null) {
17018                                                                                                        modelFactory.createTableColumn(tableModel, columnName, false);
17019                                                                                                }
17020                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17021                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
17022                                                                                                                resultColumn.bindStarLinkColumn(
17023                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
17024                                                                                                        }
17025                                                                                                }
17026                                                                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
17027                                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(table);
17028                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17029                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
17030                                                                                                                if (item.hasStarLinkColumn()) {
17031                                                                                                                        for (TObjectName starLinkColumn : item
17032                                                                                                                                        .getStarLinkColumnList()) {
17033                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
17034                                                                                                                        }
17035                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
17036                                                                                                                        resultColumn.bindStarLinkColumn(
17037                                                                                                                                        (TObjectName) item.getColumnObject());
17038                                                                                                                } else if (item.getColumnObject() instanceof TResultColumn) {
17039                                                                                                                        TResultColumn queryTableColumn = (TResultColumn) item
17040                                                                                                                                        .getColumnObject();
17041                                                                                                                        TObjectName tableColumnObject = queryTableColumn
17042                                                                                                                                        .getFieldAttr();
17043                                                                                                                        if (tableColumnObject != null) {
17044                                                                                                                                resultColumn.bindStarLinkColumn(tableColumnObject);
17045                                                                                                                        } else if (queryTableColumn.getAliasClause() != null) {
17046                                                                                                                                resultColumn.bindStarLinkColumn(queryTableColumn
17047                                                                                                                                                .getAliasClause().getAliasName());
17048                                                                                                                        }
17049                                                                                                                }
17050                                                                                                        }
17051                                                                                                }
17052                                                                                        }
17053                                                                                        continue;
17054                                                                                }
17055                                                                                resultColumn.bindStarLinkColumn(columnName);
17056                                                                        }
17057                                                                }
17058                                                        }
17059                                                }
17060                                                else {
17061                                                        ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
17062                                                }
17063                                                
17064                                                analyzeResultColumn(column, EffectType.select);
17065
17066                                        }
17067                                        
17068                                        if (queryModel instanceof ResultSet) {
17069                                                boolean isDetermined = true;
17070                                                ResultSet resultSet = (ResultSet) queryModel;
17071                                                for (ResultColumn column : resultSet.getColumns()) {
17072                                                        if (column.getName().endsWith("*")) {
17073                                                                isDetermined = false;
17074                                                                break;
17075                                                        }
17076                                                }
17077                                                if (isDetermined) {
17078                                                        resultSet.setDetermined(isDetermined);
17079                                                }
17080                                        }
17081                                }
17082                        }
17083
17084                
17085                        analyzeSelectIntoClause(stmt);
17086                
17087
17088                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
17089                                for (int i = 0; i < stmt.getJoins().size(); i++) {
17090                                        TJoin join = stmt.getJoins().getJoin(i);
17091                                        ResultSet topResultSet = (ResultSet) modelManager.getModel(stmt);
17092                                        if (join.getJoinItems() != null && join.getJoinItems().size() > 0) {
17093                                                for (int k = 0; k < join.getJoinItems().size(); k++) {
17094                                                        TTable table = join.getJoinItems().getJoinItem(k).getTable();
17095                                                        if (table != null && table.getSubquery() != null) {
17096                                                                
17097                                                                ResultSet joinResultSet = (ResultSet) modelManager.getModel(table.getSubquery());
17098                                                                for (int x = 0; x < joinResultSet.getColumns().size(); x++) {
17099                                                                        ResultColumn sourceColumn = joinResultSet.getColumns().get(x);
17100                                                                        ResultColumn resultColumn = matchResultColumn(topResultSet.getColumns(),
17101                                                                                        sourceColumn);
17102                                                                        if (resultColumn != null
17103                                                                                        && resultColumn.getColumnObject() instanceof TResultColumn) {
17104                                                                                TResultColumn column = (TResultColumn) resultColumn.getColumnObject();
17105                                                                                if (column.getAliasClause() == null && column.getFieldAttr() != null) {
17106                                                                                        TObjectName resultObject = column.getFieldAttr();
17107                                                                                        if (resultObject.getSourceTable() == null
17108                                                                                                        || resultObject.getSourceTable().equals(table)) {
17109                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
17110                                                                                                                .createDataFlowRelation();
17111                                                                                                combinedQueryRelation.setEffectType(EffectType.select);
17112                                                                                                combinedQueryRelation
17113                                                                                                                .setTarget(new ResultColumnRelationshipElement(resultColumn));
17114                                                                                                combinedQueryRelation
17115                                                                                                                .addSource(new ResultColumnRelationshipElement(sourceColumn));
17116                                                                                        }
17117                                                                                }
17118                                                                        }
17119                                                                }
17120                                                        }
17121                                                        
17122                                                        if(join.getJoinItems().getJoinItem(k).getJoin()!=null) {
17123                                                                analyzeJoin(join.getJoinItems().getJoinItem(k).getJoin(), EffectType.select);
17124                                                        }
17125                                                }
17126                                        }
17127                                        analyzeJoin(join, EffectType.select);
17128                                }
17129                        }
17130
17131                        if (stmt.getWhereClause() != null) {
17132                                TExpression expr = stmt.getWhereClause().getCondition();
17133                                if (expr != null) {
17134                                        analyzeFilterCondition(null, expr, null, JoinClauseType.where, EffectType.select);
17135                                }
17136                        }
17137
17138                        stmtStack.pop();
17139                }
17140        }
17141
17142        protected TObjectNameList getTableLinkedColumns(TTable table) {
17143                if(structObjectMap.containsKey(table)) {
17144                        return structObjectMap.get(table);
17145                }
17146                return table.getLinkedColumns();
17147        }
17148
17149        protected boolean isTopResultSet(TSelectSqlStatement stmt) {
17150                TCustomSqlStatement parent = stmt.getParentStmt();
17151                if (parent == null)
17152                        return true;
17153                if (parent instanceof TMssqlReturn) {
17154                        return true;
17155                }
17156                if (parent instanceof TReturnStmt) {
17157                        return true;
17158                }
17159                if (parent instanceof TCommonBlock) {
17160                        TCommonBlock block = (TCommonBlock) parent;
17161                        if (block.getStatements() != null) {
17162                                for (int i = 0; i < block.getStatements().size(); i++) {
17163                                        TCustomSqlStatement child = block.getStatements().get(i);
17164                                        if(stmt == child) {
17165                                                return true;
17166                                        }
17167                                }
17168                        }
17169                }
17170                if (parent instanceof TMssqlBlock) {
17171                        TMssqlBlock block = (TMssqlBlock) parent;
17172                        if (block.getStatements() != null) {
17173                                for (int i = 0; i < block.getStatements().size(); i++) {
17174                                        TCustomSqlStatement child = block.getStatements().get(i);
17175                                        if(stmt == child) {
17176                                                return true;
17177                                        }
17178                                }
17179                        }
17180                }
17181                if (parent instanceof TStoredProcedureSqlStatement) {
17182                        TStoredProcedureSqlStatement block = (TStoredProcedureSqlStatement) parent;
17183                        if (block.getStatements() != null) {
17184                                for (int i = 0; i < block.getStatements().size(); i++) {
17185                                        TCustomSqlStatement child = block.getStatements().get(i);
17186                                        if(child == stmt) {
17187                                                return true;
17188                                        }
17189                                        if (child instanceof TReturnStmt) {
17190                                                TReturnStmt returnStmt = (TReturnStmt) child;
17191                                                if (returnStmt.getStatements() != null) {
17192                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
17193                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
17194                                                                if(child1 == stmt) {
17195                                                                        return true;
17196                                                                }
17197                                                        }
17198                                                }
17199                                        }
17200                                        if (child instanceof TMssqlReturn) {
17201                                                TMssqlReturn returnStmt = (TMssqlReturn) child;
17202                                                if (returnStmt.getStatements() != null) {
17203                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
17204                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
17205                                                                if(child1 == stmt) {
17206                                                                        return true;
17207                                                                }
17208                                                        }
17209                                                }
17210                                        }
17211                                }
17212                        }
17213                }
17214                return false;
17215        }
17216
17217        protected void analyzeTableSubquery(TTable table) {
17218                if(table.getSubquery()!=null) {
17219                        QueryTable queryTable = modelFactory.createQueryTable(table);
17220                        TSelectSqlStatement subquery = table.getSubquery();
17221                        analyzeSelectStmt(subquery);
17222
17223                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
17224
17225                        if (resultSetModel != null && resultSetModel != queryTable
17226                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
17227                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
17228                                impactRelation.setEffectType(EffectType.select);
17229                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
17230                                                resultSetModel.getRelationRows()));
17231                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
17232                                                queryTable.getRelationRows()));
17233                        }
17234
17235                        if (resultSetModel != null && resultSetModel != queryTable
17236                                        && queryTable.getTableObject().getAliasClause() != null
17237                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
17238                                for (int j = 0; j < queryTable.getColumns().size()
17239                                                && j < resultSetModel.getColumns().size(); j++) {
17240                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
17241                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
17242
17243                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
17244                                        queryRalation.setEffectType(EffectType.select);
17245                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
17246                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
17247                                }
17248                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
17249                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
17250                                                .getModel(subquery);
17251                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
17252                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
17253                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
17254                                                        sourceColumn);
17255                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
17256                                                targetColumn.bindStarLinkColumn(starLinkColumn);
17257                                        }
17258                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
17259                                        selectSetRalation.setEffectType(EffectType.select);
17260                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
17261                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
17262                                }
17263                        }
17264                }
17265        }
17266
17267        private ResultColumn getPivotedTableColumn(TPivotedTable pivotedTable, TObjectName columnName) {
17268                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
17269                if (pivotedTable.getPivotClause() != null) {
17270                        pivotClauses.add(pivotedTable.getPivotClause());
17271                }
17272                if (pivotedTable.getPivotClauseList() != null) {
17273                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
17274                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
17275                        }
17276                }
17277                for (TPivotClause clause : pivotClauses) {
17278                        Object model = modelManager.getModel(clause);
17279                        if (model instanceof PivotedTable) {
17280                                PivotedTable pivotedTableModel = (PivotedTable) model;
17281                                if (pivotedTableModel.getColumns() != null) {
17282                                        for (ResultColumn column : pivotedTableModel.getColumns()) {
17283                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
17284                                                                getColumnName(SQLUtil.trimColumnStringQuote(column.getName())))) {
17285                                                        return column;
17286                                                }
17287                                        }
17288                                }
17289                        }
17290                }
17291                return null;
17292        }
17293
17294        private void analyzeHiveTransformClause(TSelectSqlStatement stmt, THiveTransformClause transformClause) {
17295                Table mapSourceTable = null;
17296                QueryTable mapQueryTable = null;
17297                if(stmt.getTables()!=null) {
17298                        for(int i=0;i<stmt.getTables().size();i++) {
17299                                TTable table = stmt.getTables().getTable(i);
17300                                if (table.getSubquery() != null) {
17301                                        if (transformClause.getTransformType() == ETransformType.ettReduce) {
17302                                                mapQueryTable = modelFactory.createQueryTable(table);
17303                                        }
17304                                        analyzeSelectStmt(table.getSubquery());
17305                                }
17306                                else {
17307                                        mapSourceTable = modelFactory.createTable(table);
17308                                }
17309                        }
17310                }
17311                
17312                if (transformClause.getTransformType() == ETransformType.ettReduce) {
17313                        modelFactory.createResultSet(stmt, false);
17314                }
17315                
17316                List<TableColumn> mapTableColumns = new ArrayList<TableColumn>();
17317                List<ResultColumn> mapResultSetColumns = new ArrayList<ResultColumn>();
17318                List<ResultColumn> redueResultSetColumns = new ArrayList<ResultColumn>();
17319                
17320                if(transformClause.getExpressionList()!=null) {
17321                        for(TExpression expression: transformClause.getExpressionList()) {
17322                                if(expression.getObjectOperand()!=null) {
17323                                        if (transformClause.getTransformType() == ETransformType.ettMap || transformClause.getTransformType() == ETransformType.ettSelect) {
17324                                                if (mapSourceTable != null) {
17325                                                        TableColumn tableColumn = modelFactory.createTableColumn(mapSourceTable,
17326                                                                        expression.getObjectOperand(), false);
17327                                                        if (tableColumn != null) {
17328                                                                mapTableColumns.add(tableColumn);
17329                                                        }
17330                                                }
17331                                        }
17332                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
17333                                                if (mapQueryTable != null) {
17334                                                        ResultColumn resultColumn = modelFactory.createResultColumn(mapQueryTable,
17335                                                                        expression.getObjectOperand(), false);
17336                                                        if (resultColumn != null) {
17337                                                                mapResultSetColumns.add(resultColumn);
17338                                                        }
17339                                                }
17340                                        }
17341                                }
17342                        }
17343                }
17344                
17345                if (transformClause.getAliasClause() != null) {
17346                        Object model = modelManager.getModel(stmt);
17347                        if (model instanceof ResultSet) {
17348                                ResultSet result = (ResultSet) model;
17349                                if (result!=null && transformClause.getAliasClause().getColumns() != null) {
17350                                        for (TObjectName column : transformClause.getAliasClause().getColumns()) {
17351                                                ResultColumn resultColumn = modelFactory.createResultColumn(result, column);
17352                                                if (resultColumn != null) {
17353                                                        if (transformClause.getTransformType() == ETransformType.ettMap 
17354                                                                        || transformClause.getTransformType() == ETransformType.ettSelect) {
17355                                                                mapResultSetColumns.add(resultColumn);
17356                                                        }
17357                                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
17358                                                                redueResultSetColumns.add(resultColumn);
17359                                                        }
17360                                                }
17361                                        }
17362                                }
17363                        }
17364                }
17365                
17366                if (transformClause.getTransformType() == ETransformType.ettMap 
17367                                || transformClause.getTransformType() == ETransformType.ettSelect) {
17368                        if (!mapTableColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
17369                                for (ResultColumn resultColumn : mapResultSetColumns) {
17370                                        for (TableColumn tableColumn : mapTableColumns) {
17371                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17372                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17373                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
17374                                                relation.setEffectType(EffectType.select);
17375                                        }
17376                                }
17377                        }
17378                }
17379                else if (transformClause.getTransformType() == ETransformType.ettReduce) {
17380                        if (!redueResultSetColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
17381                                for (ResultColumn reduceResultColumn : redueResultSetColumns) {
17382                                        for (ResultColumn mapResultColumn : mapResultSetColumns) {
17383                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17384                                                relation.setTarget(new ResultColumnRelationshipElement(reduceResultColumn));
17385                                                relation.addSource(new ResultColumnRelationshipElement(mapResultColumn));
17386                                                relation.setEffectType(EffectType.select);
17387                                        }
17388                                }
17389                        }
17390                }
17391        }
17392
17393        protected boolean isStructColumn(TObjectName columnName) {
17394                return columnName.getSourceTable() != null && columnName.getSourceTable().getAliasClause() != null
17395                                && columnName.getSourceTable().getUnnestClause() != null
17396                                && DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
17397                                                getColumnName(columnName.getSourceTable().getAliasClause().getAliasName()));
17398        }
17399
17400        private TObjectName getObjectName(ResultColumn column) {
17401                if (column.getColumnObject() instanceof TResultColumn) {
17402                        TResultColumn resultColumn = (TResultColumn) column.getColumnObject();
17403                        if (resultColumn.getAliasClause() != null && resultColumn.getAliasClause().getAliasName() != null) {
17404                                return resultColumn.getAliasClause().getAliasName();
17405                        }
17406                        if (resultColumn.getFieldAttr() != null) {
17407                                return resultColumn.getFieldAttr();
17408                        }
17409                        if (resultColumn.getExpr() != null
17410                                        && resultColumn.getExpr().getExpressionType() == EExpressionType.simple_object_name_t) {
17411                                return resultColumn.getExpr().getObjectOperand();
17412                        }
17413                } else if (column.getColumnObject() instanceof TObjectName) {
17414                        return (TObjectName) column.getColumnObject();
17415                }
17416                return null;
17417        }
17418
17419        private boolean isShowTopSelectResultSet() {
17420                if (option.isSimpleOutput() && !option.isSimpleShowTopSelectResultSet())
17421                        return false;
17422                return true;
17423        }
17424
17425        private void analyzeSelectIntoClause(TSelectSqlStatement stmt) {
17426                if (stmt.getParentStmt() instanceof TSelectSqlStatement) {
17427                        return;
17428                }
17429                
17430                TableColumn oracleIntoTableColumn = null;
17431                
17432                TIntoClause intoClause = stmt.getIntoClause();
17433                
17434                TSelectSqlStatement leftStmt = DlineageUtil.getLeftStmt(stmt);
17435                
17436                if (intoClause == null && leftStmt != null) {
17437                        intoClause = leftStmt.getIntoClause();
17438                }
17439                
17440                if (intoClause != null) {
17441                        List<TObjectName> tableNames = new ArrayList<TObjectName>();
17442                        if (intoClause.getExprList() != null) {
17443                                for (int j = 0; j < intoClause.getExprList().size(); j++) {
17444                                        TObjectName tableName = intoClause.getExprList().getExpression(j).getObjectOperand();
17445                                        if (tableName != null) {
17446                                                if (tableName.toString().startsWith(":") && option.getVendor() == EDbVendor.dbvoracle
17447                                                                && tableName.getDbObjectType() == EDbObjectType.column) {
17448                                                        TObjectName tableAlias = new TObjectName();
17449                                                        tableAlias.setString(tableName.getTableString());
17450                                                        tableNames.add(tableAlias);
17451                                                        TTable sourceTable = tableName.getSourceTable();
17452                                                        Table sourceTableModel = modelFactory.createTable(sourceTable, tableAlias);
17453                                                        oracleIntoTableColumn = modelFactory.createTableColumn(sourceTableModel, tableName, true);
17454                                                } else {
17455                                                        if (tableName != null) {
17456                                                                tableNames.add(tableName);
17457                                                        }
17458                                                }
17459                                        }
17460                                        else if(intoClause.getExprList().getExpression(j).getFunctionCall()!=null) {
17461                                                TObjectName variableName = intoClause.getExprList().getExpression(j).getFunctionCall().getFunctionName();
17462                                                tableNames.add(variableName);
17463                                                Variable variable = modelFactory.createVariable(variableName);
17464                                                variable.setSubType(SubType.record);
17465                                                TObjectName variableProperties = new TObjectName();
17466                                                variableProperties.setString("*");
17467                                                modelFactory.createTableColumn(variable, variableProperties, true);
17468                                        }
17469                                }
17470                        } else if (intoClause.getVariableList() != null) {
17471                                for (int j = 0; j < intoClause.getVariableList().size(); j++) {
17472                                        TObjectName tableName = intoClause.getVariableList().getObjectName(j);
17473                                        if (tableName != null) {
17474                                                tableNames.add(tableName);
17475                                        }
17476                                }
17477                        } else if (intoClause.getIntoName() != null) {
17478                                tableNames.add(intoClause.getIntoName());
17479                        }
17480
17481                        ResultSet queryModel = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
17482                        if (stmt.getSetOperatorType() != ESetOperatorType.none) {
17483                                queryModel = (ResultSet) modelManager.getModel(stmt);
17484                        }
17485                        for (int j = 0; j < tableNames.size(); j++) {
17486                                TObjectName tableName = tableNames.get(j);
17487                                if (tableName.getColumnNameOnly().startsWith("@")
17488                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
17489                                        continue;
17490                                }
17491
17492                                if (tableName.getColumnNameOnly().startsWith(":")
17493                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
17494                                        continue;
17495                                }
17496
17497                                Table tableModel;
17498                                TableColumn variableColumn = null;
17499
17500                                if (tableName.getDbObjectType() == EDbObjectType.variable) {
17501                                        if (tableName.toString().indexOf(".") != -1) {
17502                                                List<String> splits = SQLUtil.parseNames(tableName.toString());
17503                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
17504                                        } else {
17505                                                tableModel = modelFactory.createVariable(tableName);
17506                                        }
17507                                        if (tableModel.getSubType() == null) {
17508                                                tableModel.setSubType(SubType.record);
17509                                        }
17510                                        if(tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
17511                                                TObjectName variableProperties = new TObjectName();
17512                                                variableProperties.setString("*");
17513                                                variableColumn = modelFactory.createTableColumn(tableModel, variableProperties, true);
17514                                        }
17515                                } else {
17516                                        tableModel = modelFactory.createTableByName(tableName);
17517                                }
17518                                
17519                                if (queryModel instanceof ResultSet && (stmt.getWhereClause() != null || hasJoin(stmt))) {
17520                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
17521                                        impactRelation.setEffectType(EffectType.insert);
17522                                        impactRelation.addSource(
17523                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)queryModel).getRelationRows()));
17524                                        impactRelation.setTarget(
17525                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
17526                                }
17527                                
17528                                Process process = modelFactory.createProcess(stmt);
17529                                tableModel.addProcess(process);
17530                                
17531                                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
17532                                        for (ResultColumn resultColumn : queryModel.getColumns()) {
17533                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17534                                                                resultColumn.getName());
17535
17536                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
17537                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
17538                                                        TSQLSchema schema = sqlenv
17539                                                                        .getSQLSchema(tableModel.getDatabase() + "." + tableModel.getSchema(), true);
17540                                                        if (schema != null) {
17541                                                                TSQLTable tempTable = schema
17542                                                                                .createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
17543                                                                tempTable.addColumn(tableColumn.getName());
17544                                                        }
17545                                                }
17546
17547                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17548                                                relation.setEffectType(EffectType.insert);
17549                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17550                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17551                                                relation.setProcess(process);
17552                                        }
17553                                        
17554                                        tableModel.setDetermined(queryModel.isDetermined());
17555                                        if(queryModel.isDetermined() && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
17556                                                tableModel.setCreateTable(true, false);
17557                                        }
17558                                        return;
17559                                }
17560
17561                                boolean isDetermined = true;
17562                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
17563                                        if (tableNames.size() > 1 && tableName.getDbObjectType() == EDbObjectType.variable) {
17564                                                if (i != j) {
17565                                                        continue;
17566                                                }
17567                                        }
17568                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
17569
17570                                        if ("*".equals(column.getColumnNameOnly()) && column.getFieldAttr() != null
17571                                                        && column.getFieldAttr().getSourceTable() != null) {
17572                                                Object model = modelManager.getModel(column);
17573                                                if(model instanceof LinkedHashMap) {
17574                                                        LinkedHashMap<String, ResultColumn> columns = (LinkedHashMap<String, ResultColumn>)model;
17575                                                        for(String key: columns.keySet()) {
17576                                                                ResultColumn sourceColumn = columns.get(key);
17577                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17578                                                                                sourceColumn.getName());
17579                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17580                                                                relation.setEffectType(EffectType.insert);
17581                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17582                                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
17583                                                                relation.setProcess(process);
17584                                                        }
17585                                                }
17586                                                else if(model instanceof ResultColumn) {
17587                                                        isDetermined = false;
17588                                                        ResultColumn resultColumn = (ResultColumn) model;
17589                                                        List<TObjectName> columns = resultColumn.getStarLinkColumnList();
17590                                                        if (columns.size() > 0) {
17591                                                                for (int k = 0; k < columns.size(); k++) {
17592        
17593                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17594                                                                                        columns.get(k));
17595        
17596                                                                        if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
17597                                                                                        && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
17598                                                                                TSQLSchema schema = sqlenv.getSQLSchema(
17599                                                                                                tableModel.getDatabase() + "." + tableModel.getSchema(), true);
17600                                                                                if (schema != null) {
17601                                                                                        TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
17602                                                                                        tempTable.addColumn(tableColumn.getName());
17603                                                                                }
17604                                                                        }
17605        
17606                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17607                                                                        relation.setEffectType(EffectType.insert);
17608                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17609                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17610                                                                        relation.setProcess(process);
17611                                                                }
17612                                                                if (resultColumn.isShowStar()) {
17613                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17614                                                                                        column.getFieldAttr());
17615                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17616                                                                        relation.setEffectType(EffectType.insert);
17617                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17618                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17619                                                                        relation.setProcess(process);
17620                                                                }
17621                                                        } else {
17622                                                                TObjectName columnObject = column.getFieldAttr();
17623                                                                if (column.getAliasClause() != null) {
17624                                                                        columnObject = column.getAliasClause().getAliasName();
17625                                                                }
17626                                                                if (columnObject != null) {
17627                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17628                                                                                        columnObject);
17629                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17630                                                                        relation.setEffectType(EffectType.insert);
17631                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17632                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17633                                                                        relation.setProcess(process);
17634                                                                } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
17635                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17636                                                                                        column.getAliasClause().getAliasName());
17637                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17638                                                                        relation.setEffectType(EffectType.insert);
17639                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17640                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17641                                                                        relation.setProcess(process);
17642                                                                }
17643                                                        }
17644                                                }
17645                                        } else {
17646                                                ResultColumn resultColumn = null;
17647
17648                                                if (queryModel instanceof QueryTable) {
17649                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
17650                                                } else if (queryModel instanceof ResultSet) {
17651                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
17652                                                } else {
17653                                                        continue;
17654                                                }
17655
17656                                                if (resultColumn == null && column.getAliasClause() != null) {
17657                                                        resultColumn = (ResultColumn) modelManager.getModel(column.getAliasClause().getAliasName());
17658                                                }
17659
17660                                                if (resultColumn != null) {
17661                                                        TObjectName columnObject = column.getFieldAttr();
17662                                                        if (column.getAliasClause() != null) {
17663                                                                columnObject = column.getAliasClause().getAliasName();
17664                                                        }
17665                                                        TableColumn tableColumn = null;
17666                                                        if (columnObject != null) {
17667                                                                if (tableModel.isVariable()) {
17668                                                                        if (variableColumn != null) {
17669                                                                                tableColumn = variableColumn;
17670                                                                        } else {
17671                                                                                tableColumn = tableModel.getColumns().get(0);
17672                                                                        }
17673                                                                } 
17674                                                                else if (oracleIntoTableColumn != null) {
17675                                                                        tableColumn = oracleIntoTableColumn;
17676                                                                }
17677                                                                else {
17678                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, columnObject);
17679                                                                        if (containStarColumn(queryModel)) {
17680                                                                                tableColumn.notBindStarLinkColumn(true);
17681                                                                        }
17682                                                                }
17683                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17684                                                                relation.setEffectType(EffectType.insert);
17685                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17686                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17687                                                                relation.setProcess(process);
17688                                                        } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
17689                                                                if (tableModel.isVariable()) {
17690                                                                        if (variableColumn != null) {
17691                                                                                tableColumn = variableColumn;
17692                                                                        } else {
17693                                                                                tableColumn = tableModel.getColumns().get(0);
17694                                                                        }
17695                                                                } else {
17696                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
17697                                                                                        column.getAliasClause().getAliasName());
17698                                                                }
17699                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17700                                                                relation.setEffectType(EffectType.insert);
17701                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17702                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17703                                                                relation.setProcess(process);
17704                                                        } else {
17705                                                                if (tableModel.isVariable()) {
17706                                                                        if (variableColumn != null) {
17707                                                                                tableColumn = variableColumn;
17708                                                                        } else {
17709                                                                                tableColumn = tableModel.getColumns().get(0);
17710                                                                        }
17711                                                                }
17712                                                                if (tableColumn != null) {
17713                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17714                                                                        relation.setEffectType(EffectType.insert);
17715                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17716                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17717                                                                        relation.setProcess(process);
17718                                                                }
17719                                                        }
17720                                                }
17721                                        }
17722                                }
17723                                tableModel.setDetermined(isDetermined);
17724                                if(isDetermined && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
17725                                        tableModel.setCreateTable(true, false);
17726                                }
17727                        }
17728                }
17729        }
17730
17731        private boolean isUnPivotedTable(TPivotedTable pivotedTable) {
17732                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
17733                        return pivotedTable.getPivotClauseList().getElement(0).getType() == TPivotClause.unpivot;
17734                } else {
17735                        TPivotClause pivotClause = pivotedTable.getPivotClause();
17736                        return pivotClause.getType() == TPivotClause.unpivot;
17737                }
17738        }
17739
17740        private void analyzeUnPivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
17741                List<Object> tables = new ArrayList<Object>();
17742                Set<Object> pivotedColumns = new HashSet<Object>();
17743                TTable fromTable = pivotedTable.getTableSource();
17744                Object table = modelManager.getModel(fromTable);
17745                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
17746                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
17747                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
17748                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
17749                        }
17750                } else {
17751                        TPivotClause pivotClause = pivotedTable.getPivotClause();
17752                        pivotClauses.add(pivotClause);
17753                }
17754
17755                for (int y = 0; y < pivotClauses.size(); y++) {
17756                        TPivotClause pivotClause = pivotClauses.get(y);
17757                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
17758                        pivotTable.setUnpivoted(true);
17759
17760                        if (pivotClause.getValueColumnList() != null) {
17761                                for (int j = 0; j < pivotClause.getValueColumnList().size(); j++) {
17762                                        modelFactory.createResultColumn(pivotTable, pivotClause.getValueColumnList().getObjectName(j));
17763                                }
17764                        }
17765                        if (pivotClause.getPivotColumnList() != null) {
17766                                for (int j = 0; j < pivotClause.getPivotColumnList().size(); j++) {
17767                                        modelFactory.createResultColumn(pivotTable, pivotClause.getPivotColumnList().getObjectName(j));
17768                                }
17769                        }
17770                        if (pivotClause.getUnpivotInClause()!=null && pivotClause.getUnpivotInClause().getItems() != null) {
17771                                for (int j = 0; j < pivotClause.getUnpivotInClause().getItems().size(); j++) {
17772                                        TObjectName columnName = pivotClause.getUnpivotInClause().getItems().getElement(j).getColumn();
17773                                        if (columnName != null) {
17774                                                if (table instanceof QueryTable) {
17775                                                        for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
17776                                                                if (getColumnName(columnName)
17777                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17778                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
17779                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17780                                                                                relation.setEffectType(EffectType.select);
17781                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17782                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17783                                                                                pivotedColumns.add(tableColumn);
17784                                                                        }
17785                                                                        break;
17786                                                                }
17787                                                        }
17788                                                } else if (table instanceof Table) {
17789                                                        for (TableColumn tableColumn : ((Table) table).getColumns()) {
17790                                                                if (getColumnName(columnName)
17791                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17792                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
17793                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17794                                                                                relation.setEffectType(EffectType.select);
17795                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17796                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
17797                                                                                pivotedColumns.add(tableColumn);
17798                                                                        }
17799                                                                        break;
17800                                                                }
17801                                                        }
17802                                                }
17803                                        } else {
17804                                                TObjectNameList columnNames = pivotClause.getUnpivotInClause().getItems().getElement(j)
17805                                                                .getColumnList();
17806                                                for (TObjectName columnName1 : columnNames) {
17807                                                        if (table instanceof QueryTable) {
17808                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
17809                                                                        if (getColumnName(columnName1).equals(
17810                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17811                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
17812                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17813                                                                                        relation.setEffectType(EffectType.select);
17814                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17815                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17816                                                                                        pivotedColumns.add(tableColumn);
17817                                                                                }
17818                                                                                break;
17819                                                                        }
17820                                                                }
17821                                                        } else if (table instanceof Table) {
17822                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
17823                                                                        if (getColumnName(columnName1).equals(
17824                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17825                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
17826                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17827                                                                                        relation.setEffectType(EffectType.select);
17828                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17829                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
17830                                                                                        pivotedColumns.add(tableColumn);
17831                                                                                }
17832                                                                                break;
17833                                                                        }
17834                                                                }
17835                                                        }
17836                                                }
17837                                        }
17838                                }
17839                        }
17840                        tables.add(pivotTable);
17841                        tables.add(table);
17842                }
17843
17844                ResultSet resultSet = modelFactory.createResultSet(stmt,
17845                                isTopResultSet(stmt) && isShowTopSelectResultSet());
17846                TResultColumnList columnList = stmt.getResultColumnList();
17847                for (int i = 0; i < columnList.size(); i++) {
17848                        TResultColumn column = columnList.getResultColumn(i);
17849                        ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
17850                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
17851                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
17852                                if (columnObject.getFieldAttr() != null) {
17853                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
17854                                                resultColumn.setShowStar(false);
17855                                                int index = 0;
17856                                                for (int k = 0; k < tables.size(); k++) {
17857                                                        Object tableItem = tables.get(k);
17858                                                        if (tableItem instanceof ResultSet) {
17859                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
17860                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
17861                                                                        if (pivotedColumns.contains(tableColumn)) {
17862                                                                                continue;
17863                                                                        }
17864                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
17865                                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
17866                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17867                                                                                relation.setEffectType(EffectType.select);
17868                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
17869                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17870                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
17871                                                                                if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
17872                                                                                        if (tableColumn.hasStarLinkColumn()) {
17873                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList().size(); z++) {
17874                                                                                                        ResultColumn resultColumn1 = modelFactory.createResultColumn(
17875                                                                                                                        (ResultSet) tableItem,
17876                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
17877                                                                                                        DataFlowRelationship relation = modelFactory
17878                                                                                                                        .createDataFlowRelation();
17879                                                                                                        relation.setEffectType(EffectType.select);
17880                                                                                                        relation.setTarget(
17881                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
17882                                                                                                        relation.addSource(
17883                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
17884                                                                                                        tableColumn.getStarLinkColumns().remove(
17885                                                                                                                        getColumnName(tableColumn.getStarLinkColumnList().get(z)));
17886                                                                                                        z--;
17887                                                                                                }
17888                                                                                        } else {
17889                                                                                                resultColumn.bindStarLinkColumn(
17890                                                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
17891                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17892                                                                                                relation.setEffectType(EffectType.select);
17893                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr()));
17894                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17895                                                                                        }
17896                                                                                } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
17897                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
17898                                                                                                        .getExpr();
17899                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
17900                                                                                                TObjectName columnName = new TObjectName();
17901                                                                                                columnName.setString(expr.toString());
17902                                                                                                resultColumn.bindStarLinkColumn(columnName);
17903                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17904                                                                                                relation.setEffectType(EffectType.select);
17905                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, columnName));
17906                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17907                                                                                        }
17908                                                                                }
17909                                                                        }
17910                                                                }
17911                                                        } else if (tableItem instanceof Table) {
17912                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
17913                                                                        if (pivotedColumns.contains(tableColumn)) {
17914                                                                                continue;
17915                                                                        }
17916                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(), index);
17917                                                                        index++;
17918                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17919                                                                        relation.setEffectType(EffectType.select);
17920                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
17921                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
17922                                                                }
17923                                                        }
17924                                                }
17925                                        } else {
17926                                                for (int k = 0; k < tables.size(); k++) {
17927                                                        Object tableItem = tables.get(k);
17928                                                        if (tableItem instanceof ResultSet) {
17929                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
17930                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
17931                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17932                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17933                                                                                relation.setEffectType(EffectType.select);
17934                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17935                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17936                                                                                break;
17937                                                                        }
17938                                                                }
17939                                                        } else if (tableItem instanceof Table) {
17940                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
17941                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
17942                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17943                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17944                                                                                relation.setEffectType(EffectType.select);
17945                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17946                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
17947                                                                                break;
17948                                                                        }
17949                                                                }
17950                                                        }
17951                                                }
17952                                        }
17953                                } else if (columnObject.getExpr() != null) {
17954                                        analyzeResultColumn(column, EffectType.select);
17955                                }
17956                        }
17957                }
17958        }
17959
17960        private void analyzePivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
17961                List<Object> tables = new ArrayList<Object>();
17962                Set<Object> pivotedColumns = new HashSet<Object>();
17963                TTable fromTable = pivotedTable.getTableSource();
17964                Object table = modelManager.getModel(fromTable);
17965                if(table == null && fromTable.getSubquery()!=null) {
17966                        table = modelFactory.createQueryTable(fromTable);
17967                        TSelectSqlStatement subquery = fromTable.getSubquery();
17968                        analyzeSelectStmt(subquery);
17969                }
17970                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
17971                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
17972                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
17973                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
17974                        }
17975                } else {
17976                        TPivotClause pivotClause = pivotedTable.getPivotClause();
17977                        pivotClauses.add(pivotClause);
17978                }
17979
17980                for (int y = 0; y < pivotClauses.size(); y++) {
17981                        TPivotClause pivotClause = pivotClauses.get(y);
17982                        List<TFunctionCall> functionCalls = new ArrayList<TFunctionCall>();
17983                        if (pivotClause.getAggregation_function() != null || pivotClause.getAggregation_function_list() != null) {
17984                                if (pivotClause.getAggregation_function() != null) {
17985                                        functionCalls.add((TFunctionCall) pivotClause.getAggregation_function());
17986                                } else if (pivotClause.getAggregation_function_list() != null) {
17987                                        for (int i = 0; i < pivotClause.getAggregation_function_list().size(); i++) {
17988                                                functionCalls.add((TFunctionCall) pivotClause.getAggregation_function_list().getResultColumn(i)
17989                                                                .getExpr().getFunctionCall());
17990                                        }
17991                                }
17992
17993                                if (functionCalls.isEmpty()) {
17994                                        return;
17995                                }
17996
17997                                if (pivotClause.getPivotColumnList() == null) {
17998                                        return;
17999                                }
18000
18001                                if (pivotClause.getPivotInClause() == null) {
18002                                        return;
18003                                }
18004
18005                                for (int x = 0; x < functionCalls.size(); x++) {
18006                                        TFunctionCall functionCall = functionCalls.get(x);
18007                                        Function function = modelFactory.createFunction(functionCall);
18008                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
18009                                                        ((TFunctionCall) functionCall).getFunctionName());
18010
18011                                        List<TExpression> expressions = new ArrayList<TExpression>();
18012                                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
18013
18014                                        for (int j = 0; j < expressions.size(); j++) {
18015                                                columnsInExpr visitor = new columnsInExpr();
18016                                                expressions.get(j).inOrderTraverse(visitor);
18017                                                List<TObjectName> objectNames = visitor.getObjectNames();
18018                                                if (objectNames == null) {
18019                                                        continue;
18020                                                }
18021                                                for (TObjectName columnName : objectNames) {
18022                                                        if (table instanceof QueryTable) {
18023                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
18024                                                                        boolean find = false;
18025                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
18026                                                                        if (tableColumn.hasStarLinkColumn()) {
18027                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
18028                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
18029                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
18030                                                                                                ResultColumn resultColumn = modelFactory
18031                                                                                                                .createResultColumn((QueryTable) table, objectName);
18032                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18033                                                                                                relation.setEffectType(EffectType.select);
18034                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18035                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18036                                                                                                pivotedColumns.add(resultColumn);
18037                                                                                                tableColumn.getStarLinkColumns()
18038                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
18039                                                                                                find = true;
18040                                                                                                break;
18041                                                                                        }
18042                                                                                }
18043                                                                        } else if (getColumnName(columnName).equals(
18044                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18045                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18046                                                                                relation.setEffectType(EffectType.select);
18047                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18048                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18049                                                                                pivotedColumns.add(tableColumn);
18050                                                                                find = true;
18051                                                                                break;
18052                                                                        } 
18053                                                                        
18054                                                                        if (!find && tableColumn.getName().endsWith("*")) {
18055                                                                                QueryTable queryTable = (QueryTable)table;
18056                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(queryTable, columnName);
18057                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18058                                                                                relation.setEffectType(EffectType.select);
18059                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18060                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18061                                                                                tableColumn.bindStarLinkColumn(columnName);
18062                                                                        }
18063                                                                }
18064                                                        } else if (table instanceof Table) {
18065                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18066                                                                        if (getColumnName(columnName).equals(
18067                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18068                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18069                                                                                relation.setEffectType(EffectType.select);
18070                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18071                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18072                                                                                pivotedColumns.add(tableColumn);
18073                                                                                break;
18074                                                                        }
18075                                                                }
18076                                                        }
18077                                                }
18078                                        }
18079
18080                                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
18081                                        pivotTable.setUnpivoted(false);
18082
18083                                        if (pivotClause.getPivotInClause().getItems() != null) {
18084                                                for (int j = 0; j < pivotClause.getPivotInClause().getItems().size(); j++) {
18085                                                        ResultColumn resultColumn = null;
18086                                                        if (pivotClause.getAggregation_function_list() != null
18087                                                                        && pivotClause.getAggregation_function_list().size() > 1) {
18088                                                                TResultColumn functionColumn = pivotClause.getAggregation_function_list()
18089                                                                                .getResultColumn(x);
18090                                                                TObjectName tableColumn = new TObjectName();
18091                                                                if (option.getVendor() == EDbVendor.dbvbigquery) {
18092                                                                        tableColumn.setString(getResultColumnString(functionColumn) + "_"
18093                                                                                        + SQLUtil.trimColumnStringQuote(getResultColumnString(
18094                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j))));
18095                                                                }
18096                                                                else {
18097                                                                        tableColumn.setString(SQLUtil
18098                                                                                        .trimColumnStringQuote(getResultColumnString(
18099                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j)))
18100                                                                                        + "_" + getResultColumnString(functionColumn));
18101                                                                }
18102                                                                resultColumn = modelFactory.createResultColumn(pivotTable, tableColumn);
18103                                                        } else {
18104                                                                resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
18105                                                                                pivotClause.getPivotInClause().getItems().getResultColumn(j), j);
18106                                                        }
18107                                                        {
18108                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18109                                                                relation.setEffectType(EffectType.select);
18110                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18111                                                                relation.addSource(new ResultColumnRelationshipElement(column));
18112                                                        }
18113                                                        {
18114                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
18115                                                                        if (table instanceof QueryTable) {
18116                                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
18117                                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
18118                                                                                        if (tableColumn.hasStarLinkColumn()) {
18119                                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
18120                                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
18121                                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
18122                                                                                                                ResultColumn resultColumn1 = modelFactory
18123                                                                                                                                .createResultColumn((QueryTable) table, objectName);
18124                                                                                                                DataFlowRelationship relation = modelFactory
18125                                                                                                                                .createDataFlowRelation();
18126                                                                                                                relation.setEffectType(EffectType.select);
18127                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18128                                                                                                                relation.addSource(
18129                                                                                                                                new ResultColumnRelationshipElement(resultColumn1));
18130                                                                                                                pivotedColumns.add(resultColumn1);
18131                                                                                                                tableColumn.getStarLinkColumns()
18132                                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
18133                                                                                                                break;
18134                                                                                                        }
18135                                                                                                }
18136                                                                                        } else if (getColumnName(columnName).equals(DlineageUtil
18137                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18138                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18139                                                                                                relation.setEffectType(EffectType.select);
18140                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18141                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18142                                                                                                pivotedColumns.add(tableColumn);
18143                                                                                                break;
18144                                                                                        }
18145                                                                                }
18146                                                                        } else if (table instanceof Table) {
18147                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18148                                                                                        if (getColumnName(columnName).equals(DlineageUtil
18149                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18150                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18151                                                                                                relation.setEffectType(EffectType.select);
18152                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18153                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18154                                                                                                pivotedColumns.add(tableColumn);
18155                                                                                                break;
18156                                                                                        }
18157                                                                                }
18158                                                                        }
18159                                                                }
18160                                                        }
18161                                                }
18162                                        } else if (pivotClause.getPivotInClause().getSubQuery() != null) {
18163                                                TSelectSqlStatement subquery = pivotClause.getPivotInClause().getSubQuery();
18164                                                analyzeSelectStmt(subquery);
18165                                                ResultSet selectSetResultSetModel = (ResultSet) modelManager.getModel(subquery);
18166                                                for (int j = 0; j < subquery.getResultColumnList().size(); j++) {
18167                                                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
18168                                                                        subquery.getResultColumnList().getResultColumn(j), j);
18169                                                        {
18170                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18171                                                                relation.setEffectType(EffectType.select);
18172                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18173                                                                relation.addSource(new ResultColumnRelationshipElement(column));
18174                                                                relation.addSource(new ResultColumnRelationshipElement(
18175                                                                                selectSetResultSetModel.getColumns().get(j)));
18176                                                        }
18177                                                        {
18178                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
18179                                                                        if (table instanceof QueryTable) {
18180                                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
18181                                                                                        if (getColumnName(columnName).equals(DlineageUtil
18182                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18183                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18184                                                                                                relation.setEffectType(EffectType.select);
18185                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18186                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18187                                                                                                pivotedColumns.add(tableColumn);
18188                                                                                                break;
18189                                                                                        }
18190                                                                                }
18191                                                                        } else if (table instanceof Table) {
18192                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18193                                                                                        if (getColumnName(columnName).equals(DlineageUtil
18194                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18195                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18196                                                                                                relation.setEffectType(EffectType.select);
18197                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18198                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18199                                                                                                pivotedColumns.add(tableColumn);
18200                                                                                                break;
18201                                                                                        }
18202                                                                                }
18203                                                                        }
18204                                                                }
18205                                                        }
18206                                                }
18207                                        }
18208                                        tables.add(pivotTable);
18209                                        tables.add(table);
18210                                }
18211                        }
18212                }
18213
18214                TPivotClause pivotClause = pivotClauses.get(0);
18215                boolean hasAlias = pivotClause.getAliasClause() != null && pivotClause.getAliasClause().getColumns() != null
18216                                && pivotClause.getAliasClause().getColumns().size() > 0;
18217                if (hasAlias) {
18218                        Alias alias = modelFactory.createAlias(pivotClause.getAliasClause());
18219                        List<TObjectName> aliasColumns = new ArrayList<TObjectName>();
18220                        int index = 0;
18221                        for (int k = 0; k < tables.size(); k++) {
18222                                Object tableItem = tables.get(k);
18223                                if (tableItem instanceof ResultSet) {
18224                                        for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18225                                                if (pivotedColumns.contains(tableColumn)) {
18226                                                        continue;
18227                                                }
18228                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
18229                                                        aliasColumns.add((TObjectName) tableColumn.getColumnObject());
18230                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18231                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
18232                                                                aliasColumns.add(((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
18233                                                        } else {
18234                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
18235                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18236                                                                        TObjectName columnName = new TObjectName();
18237                                                                        columnName.setString(expr.toString());
18238                                                                        aliasColumns.add(columnName);
18239                                                                }
18240                                                        }
18241                                                }
18242                                        }
18243                                } else if (tableItem instanceof Table) {
18244                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18245                                                if (pivotedColumns.contains(tableColumn)) {
18246                                                        continue;
18247                                                }
18248                                                aliasColumns.add(index, (TObjectName) tableColumn.getColumnObject());
18249                                                index++;
18250                                        }
18251                                }
18252                        }
18253
18254                        IndexedLinkedHashMap<String, ResultColumn> aliasColumnMap = new IndexedLinkedHashMap<String, ResultColumn>();
18255                        int diffCount = pivotClause.getAliasClause().getColumns().size() -  aliasColumns.size();
18256                        for (int k = 0; k < pivotClause.getAliasClause().getColumns().size(); k++) {
18257                                if (pivotClause.getAliasClause().getColumns().size() > aliasColumns.size()) {
18258                                        if (k < diffCount) {
18259                                                continue;
18260                                        }
18261                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
18262                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
18263                                        if ((k - diffCount) < aliasColumns.size()) {
18264                                                aliasColumnMap.put(aliasColumns.get(k - diffCount).toString(), resultColumn);
18265                                        }
18266                                } else {
18267                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
18268                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
18269                                        if (k < aliasColumns.size()) {
18270                                                aliasColumnMap.put(aliasColumns.get(k).toString(), resultColumn);
18271                                        }
18272                                }
18273                        }
18274
18275                        for (int k = 0; k < tables.size(); k++) {
18276                                Object tableItem = tables.get(k);
18277                                if (tableItem instanceof ResultSet) {
18278                                        int resultColumnSize = ((ResultSet) tableItem).getColumns().size();
18279                                        for (int x = 0; x < resultColumnSize; x++) {
18280                                                ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
18281                                                if (pivotedColumns.contains(tableColumn)) {
18282                                                        continue;
18283                                                }
18284                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
18285                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18286                                                        relation.setEffectType(EffectType.select);
18287                                                        relation.setTarget(new ResultColumnRelationshipElement(
18288                                                                        aliasColumnMap.get(tableColumn.getColumnObject().toString())));
18289                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18290                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18291                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
18292                                                                ResultColumn targetColumn = aliasColumnMap.get(
18293                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr().toString());
18294                                                                if(targetColumn == null) {
18295                                                                        continue;
18296                                                                }
18297                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18298                                                                relation.setEffectType(EffectType.select);
18299                                                                relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
18300                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18301                                                        } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
18302                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
18303                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18304                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18305                                                                        relation.setEffectType(EffectType.select);
18306                                                                        ResultColumn targetColumn = (ResultColumn) aliasColumnMap
18307                                                                                        .getValueAtIndex(aliasColumnMap.size() - resultColumnSize + x);
18308                                                                        relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
18309                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18310                                                                }
18311                                                        }
18312                                                }
18313                                        }
18314                                } else if (tableItem instanceof Table) {
18315                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18316                                                if (pivotedColumns.contains(tableColumn)) {
18317                                                        continue;
18318                                                }
18319                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18320                                                relation.setEffectType(EffectType.select);
18321                                                relation.setTarget(new ResultColumnRelationshipElement(
18322                                                                aliasColumnMap.get(tableColumn.getColumnObject().toString())));
18323                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18324                                        }
18325                                }
18326                        }
18327
18328                        ResultSet resultSet = modelFactory.createResultSet(stmt,
18329                                        isTopResultSet(stmt) && isShowTopSelectResultSet());
18330                        TResultColumnList columnList = stmt.getResultColumnList();
18331                        for (int i = 0; i < columnList.size(); i++) {
18332                                TResultColumn column = columnList.getResultColumn(i);
18333                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
18334                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
18335                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
18336                                        if (columnObject.getFieldAttr() != null) {
18337                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
18338                                                        resultColumn.setShowStar(false);
18339                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
18340                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
18341                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18342                                                                relation.setEffectType(EffectType.select);
18343                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18344                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18345                                                        }
18346                                                } else {
18347                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
18348                                                                if (getColumnName(columnObject.getFieldAttr())
18349                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18350                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18351                                                                        relation.setEffectType(EffectType.select);
18352                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18353                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18354                                                                        break;
18355                                                                }
18356                                                        }
18357                                                }
18358                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
18359                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
18360                                                for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
18361                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
18362                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18363                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18364                                                                relation.setEffectType(EffectType.select);
18365                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18366                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18367                                                                break;
18368                                                        }
18369                                                }
18370                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
18371                                                        .getExpressionType() == EExpressionType.function_t) {
18372                                                Function function = (Function) createFunction(columnObject.getExpr().getFunctionCall());
18373                                                for (ResultColumn arg : function.getColumns()) {
18374                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18375                                                        relation.setEffectType(EffectType.select);
18376                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18377                                                        relation.addSource(new ResultColumnRelationshipElement(arg));
18378                                                }
18379                                        }
18380                                }
18381                        }
18382                } else {
18383                        ResultSet resultSet = modelFactory.createResultSet(stmt, isTopResultSet(stmt));
18384                        TResultColumnList columnList = stmt.getResultColumnList();
18385                        for (int i = 0; i < columnList.size(); i++) {
18386                                TResultColumn column = columnList.getResultColumn(i);
18387                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
18388                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
18389                                        boolean fromFunction = false;
18390                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
18391                                        TObjectName resultColumnFieldAttr = columnObject.getFieldAttr();
18392                                        List<TObjectName> resultColumnNames = new ArrayList<TObjectName>();
18393                                        if (resultColumnFieldAttr != null) {
18394                                                resultColumnNames.add(resultColumnFieldAttr);
18395                                        } else if (columnObject.getExpr() != null
18396                                                        && column.getExpr().getExpressionType() == EExpressionType.function_t) {
18397                                                extractFunctionObjectNames(column.getExpr().getFunctionCall(), resultColumnNames);
18398                                                fromFunction = true;
18399                                        }
18400                                        
18401                                        if (!resultColumnNames.isEmpty()) {
18402                                                for (TObjectName resultColumnName : resultColumnNames) {
18403                                                        if ("*".equals(getColumnName(resultColumnName))) {
18404                                                                resultColumn.setShowStar(false);
18405                                                                int index = 0;
18406                                                                for (int k = 0; k < tables.size(); k++) {
18407                                                                        Object tableItem = tables.get(k);
18408                                                                        if (tableItem instanceof ResultSet && !(tableItem instanceof QueryTable)) {
18409                                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
18410                                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
18411                                                                                        if (pivotedColumns.contains(tableColumn)) {
18412                                                                                                continue;
18413                                                                                        }
18414                                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
18415                                                                                                resultColumn.bindStarLinkColumn(
18416                                                                                                                (TObjectName) tableColumn.getColumnObject());
18417                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18418                                                                                                relation.setEffectType(EffectType.select);
18419                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18420                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18421                                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18422                                                                                                if (((TResultColumn) tableColumn.getColumnObject())
18423                                                                                                                .getFieldAttr() != null) {
18424                                                                                                        if (tableColumn.hasStarLinkColumn()) {
18425                                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList()
18426                                                                                                                                .size(); z++) {
18427                                                                                                                        ResultColumn resultColumn1 = modelFactory
18428                                                                                                                                        .createResultColumn((ResultSet) tableItem,
18429                                                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
18430                                                                                                                        DataFlowRelationship relation = modelFactory
18431                                                                                                                                        .createDataFlowRelation();
18432                                                                                                                        relation.setEffectType(EffectType.select);
18433                                                                                                                        relation.setTarget(
18434                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18435                                                                                                                        relation.addSource(
18436                                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
18437                                                                                                                        tableColumn.getStarLinkColumns().remove(getColumnName(
18438                                                                                                                                        tableColumn.getStarLinkColumnList().get(z)));
18439                                                                                                                        z--;
18440                                                                                                                }
18441                                                                                                        } else {
18442                                                                                                                resultColumn.bindStarLinkColumn(
18443                                                                                                                                ((TResultColumn) tableColumn.getColumnObject())
18444                                                                                                                                                .getFieldAttr());
18445                                                                                                                DataFlowRelationship relation = modelFactory
18446                                                                                                                                .createDataFlowRelation();
18447                                                                                                                relation.setEffectType(EffectType.select);
18448                                                                                                                relation.setTarget(
18449                                                                                                                                new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject())
18450                                                                                                                                                .getFieldAttr()));
18451                                                                                                                relation.addSource(
18452                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
18453                                                                                                        }
18454                                                                                                } else if (((TResultColumn) tableColumn.getColumnObject())
18455                                                                                                                .getExpr() != null) {
18456                                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
18457                                                                                                                        .getExpr();
18458                                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18459                                                                                                                TObjectName columnName = new TObjectName();
18460                                                                                                                columnName.setString(expr.toString());
18461                                                                                                                resultColumn.bindStarLinkColumn(columnName);
18462                                                                                                                DataFlowRelationship relation = modelFactory
18463                                                                                                                                .createDataFlowRelation();
18464                                                                                                                relation.setEffectType(EffectType.select);
18465                                                                                                                relation.setTarget(
18466                                                                                                                                new ResultColumnRelationshipElement(resultColumn, columnName));
18467                                                                                                                relation.addSource(
18468                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
18469                                                                                                        }
18470                                                                                                }
18471                                                                                        }
18472                                                                                }
18473                                                                        } else if (tableItem instanceof Table) {
18474                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18475                                                                                        if (pivotedColumns.contains(tableColumn)) {
18476                                                                                                continue;
18477                                                                                        }
18478                                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(),
18479                                                                                                        index);
18480                                                                                        index++;
18481                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18482                                                                                        relation.setEffectType(EffectType.select);
18483                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18484                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18485                                                                                }
18486                                                                        } else if (tableItem instanceof QueryTable) {
18487                                                                                for (ResultColumn tableColumn : ((QueryTable) tableItem).getColumns()) {
18488                                                                                        if (pivotedColumns.contains(tableColumn)) {
18489                                                                                                continue;
18490                                                                                        }
18491                                                                                        TObjectName column1 = new TObjectName();
18492                                                                                        column1.setString(tableColumn.getName());
18493                                                                                        resultColumn.bindStarLinkColumn(column1, index);
18494                                                                                        index++;
18495                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18496                                                                                        relation.setEffectType(EffectType.select);
18497                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, column1));
18498                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn), false);
18499                                                                                }
18500                                                                        }
18501                                                                }
18502                                                        } else {
18503                                                                ResultColumn pivotedTableColumn = getPivotedTableColumn(pivotedTable, resultColumnName);
18504                                                                        if (pivotedTableColumn != null) {
18505                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18506                                                                                relation.setEffectType(EffectType.select);
18507                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18508                                                                                relation.addSource(new ResultColumnRelationshipElement(pivotedTableColumn));
18509                                                                        } else {
18510                                                                                for (int k = 0; k < tables.size(); k++) {
18511                                                                                        Object tableItem = tables.get(k);
18512                                                                                        if (tableItem instanceof ResultSet) {
18513                                                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18514                                                                                                        if (DlineageUtil
18515                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()).equals(getColumnName(resultColumnName))) {
18516                                                                                                                if (fromFunction) {
18517                                                                                                                        Function function = (Function)createPivotedFunction(column.getExpr().getFunctionCall(), tableColumn);
18518                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18519                                                                                                                        relation.setEffectType(EffectType.select);
18520                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18521                                                                                                                        if (function.getColumns() != null && !function.getColumns().isEmpty()) {
18522                                                                                                                                for (ResultColumn functionColumn : function.getColumns()) {
18523                                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(functionColumn));
18524                                                                                                                                }
18525                                                                                                                        }
18526                                                                                                                } else {
18527                                                                                                                        DataFlowRelationship relation = modelFactory
18528                                                                                                                                        .createDataFlowRelation();
18529                                                                                                                        relation.setEffectType(EffectType.select);
18530                                                                                                                        relation.setTarget(
18531                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18532                                                                                                                        relation.addSource(
18533                                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
18534                                                                                                                }
18535                                                                                                                break;
18536                                                                                                        }
18537                                                                                                }
18538                                                                                        } else if (tableItem instanceof Table) {
18539                                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18540                                                                                                        if (getColumnName(resultColumnName).equals(DlineageUtil
18541                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18542                                                                                                                DataFlowRelationship relation = modelFactory
18543                                                                                                                                .createDataFlowRelation();
18544                                                                                                                relation.setEffectType(EffectType.select);
18545                                                                                                                relation.setTarget(
18546                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
18547                                                                                                                relation.addSource(
18548                                                                                                                                new TableColumnRelationshipElement(tableColumn));
18549                                                                                                                break;
18550                                                                                                        }
18551                                                                                                }
18552                                                                                        }
18553                                                                                }
18554                                                                        }
18555                                                                }
18556                                                }
18557                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
18558                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
18559                                                for (int k = 0; k < tables.size(); k++) {
18560                                                        Object tableItem = tables.get(k);
18561                                                        if (tableItem instanceof ResultSet) {
18562                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18563                                                                        if (columnObject.getExpr().getRightOperand().getObjectOperand() != null
18564                                                                                        && getColumnName(
18565                                                                                                        columnObject.getExpr().getRightOperand().getObjectOperand())
18566                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(
18567                                                                                                                                        tableColumn.getName()))) {
18568                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18569                                                                                relation.setEffectType(EffectType.select);
18570                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18571                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18572                                                                                break;
18573                                                                        } else if (columnObject.getExpr().getRightOperand()
18574                                                                                        .getExpressionType() == EExpressionType.function_t) {
18575                                                                                List<TExpression> expressions = new ArrayList<TExpression>();
18576                                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(),
18577                                                                                                columnObject.getExpr().getRightOperand().getFunctionCall());
18578                                                                                for (int j = 0; j < expressions.size(); j++) {
18579                                                                                        columnsInExpr visitor = new columnsInExpr();
18580                                                                                        expressions.get(j).inOrderTraverse(visitor);
18581                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
18582                                                                                        if (objectNames == null) {
18583                                                                                                continue;
18584                                                                                        }
18585                                                                                        for (TObjectName columnName : objectNames) {
18586                                                                                                if (getColumnName(columnName).equals(DlineageUtil
18587                                                                                                                .getIdentifierNormalColumnName(tableColumn.getName()))) {
18588                                                                                                        DataFlowRelationship relation = modelFactory
18589                                                                                                                        .createDataFlowRelation();
18590                                                                                                        relation.setEffectType(EffectType.select);
18591                                                                                                        relation.setTarget(
18592                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18593                                                                                                        relation.addSource(
18594                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
18595                                                                                                        break;
18596                                                                                                }
18597                                                                                        }
18598                                                                                }
18599                                                                        }
18600                                                                }
18601                                                        } else if (tableItem instanceof Table) {
18602                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18603                                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
18604                                                                                        .equals(DlineageUtil
18605                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18606                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18607                                                                                relation.setEffectType(EffectType.select);
18608                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18609                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18610                                                                                break;
18611                                                                        }
18612                                                                }
18613                                                        }
18614                                                }
18615                                        }
18616                                }
18617                        }
18618                }
18619
18620                analyzeSelectIntoClause(stmt);
18621        }
18622
18623        private Function createPivotedFunction(TFunctionCall functionCall, ResultColumn sourceColumn) {
18624                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
18625                ResultColumn column = modelFactory.createFunctionResultColumn(function,
18626                                ((TFunctionCall) functionCall).getFunctionName());
18627                if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
18628                        // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
18629                        // COUNT特殊处理，不和参数关联
18630                        if (option.isShowCountTableColumn()) {
18631                                analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
18632                        }
18633                } else {
18634                        analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
18635                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
18636                        if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
18637                                ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
18638                                if (functionTableModel.getColumns() != null) {
18639                                        for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
18640                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18641                                                        relation.setEffectType(EffectType.select);
18642                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
18643                                                        relation.addSource(new ResultColumnRelationshipElement(
18644                                                                        functionTableModel.getColumns().get(j)));
18645                                        }
18646                                }
18647                        }
18648                }
18649                return function;
18650        }
18651
18652        private void analyzePivotedFunctionArgumentsDataFlowRelation(ResultColumn column, TFunctionCall functionCall,
18653                        ResultColumn sourceColumn) {
18654                List<TExpression> directExpressions = new ArrayList<TExpression>();
18655                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
18656
18657                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
18658
18659                for (int j = 0; j < directExpressions.size(); j++) {
18660                        columnsInExpr visitor = new columnsInExpr();
18661                        directExpressions.get(j).inOrderTraverse(visitor);
18662
18663                        List<TObjectName> objectNames = visitor.getObjectNames();
18664                        List<TParseTreeNode> constants = visitor.getConstants();
18665
18666                        if (objectNames != null) {
18667                                for (TObjectName name : objectNames) {
18668                                        if (DlineageUtil.compareColumnIdentifier(name.toString(), sourceColumn.getName())) {
18669                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18670                                                relation.setEffectType(EffectType.select);
18671                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18672                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
18673                                        }
18674                                }
18675                        }
18676                        if (constants != null) {
18677                                for (TParseTreeNode name : constants) {
18678                                        if (name.toString().equals(sourceColumn.getName())) {
18679                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18680                                                relation.setEffectType(EffectType.select);
18681                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18682                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
18683                                        }
18684                                }
18685                        }
18686                }
18687        }
18688        
18689        private void extractFunctionObjectNames(TFunctionCall functionCall, List<TObjectName> resultColumnNames) {
18690                List<TExpression> directExpressions = new ArrayList<TExpression>();
18691                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
18692
18693                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
18694
18695                for (int j = 0; j < directExpressions.size(); j++) {
18696                        columnsInExpr visitor = new columnsInExpr();
18697                        directExpressions.get(j).inOrderTraverse(visitor);
18698
18699                        List<TObjectName> objectNames = visitor.getObjectNames();
18700                        List<TParseTreeNode> functions = visitor.getFunctions();
18701                        List<TParseTreeNode> constants = visitor.getConstants(); 
18702
18703                        if (objectNames != null) {
18704                                resultColumnNames.addAll(objectNames);
18705                        }
18706                        
18707                        if (constants != null) {
18708                                for(TParseTreeNode item: constants) {
18709                                        if(item instanceof TConstant && ((TConstant) item).getLiteralType().getText().equals(ELiteralType.string_et.getText())) {
18710                                                TObjectName object = new TObjectName();
18711                                                object.setString(item.toString());
18712                                                resultColumnNames.add(object);
18713                                        }
18714                                }
18715                        }
18716
18717                        if (functions != null && !functions.isEmpty()) {
18718                                for (TParseTreeNode function : functions) {
18719                                        if (function instanceof TFunctionCall) {
18720                                                extractFunctionObjectNames((TFunctionCall) function, resultColumnNames);
18721                                        }
18722                                }
18723                        }
18724                }
18725        }
18726
18727        private String getResultColumnString(TResultColumn resultColumn) {
18728                if (resultColumn.getAliasClause() != null) {
18729                        return resultColumn.getAliasClause().toString();
18730                }
18731                return resultColumn.toString();
18732        }
18733
18734        private void analyzeBigQueryUnnest(TSelectSqlStatement stmt, TTable table) {
18735                Table unnestTable = modelFactory.createTableFromCreateDDL(table, false, getTempTableName(table));
18736                unnestTable.setSubType(SubType.unnest);
18737                TUnnestClause clause = table.getUnnestClause();
18738                TExpression arrayExpr = clause.getArrayExpr();
18739                if (arrayExpr == null){
18740                        if (clause.getColumns() != null) {
18741                                for (TObjectName column : clause.getColumns()) {
18742                                        if (clause.getDerivedColumnList() != null) {
18743                                                unnestTable.setCreateTable(true);
18744                                                for (int i = 0; i < clause.getDerivedColumnList().size(); i++) {
18745                                                        TObjectName columnName = new TObjectName();
18746                                                        columnName.setString(column.getColumnNameOnly() + "."
18747                                                                        + clause.getDerivedColumnList().getObjectName(i).getColumnNameOnly());
18748                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
18749                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
18750                                                        columns.add(column);
18751                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18752                                                }
18753                                        }
18754                                        else {
18755                                                unnestTable.setCreateTable(true);
18756                                                boolean find = false;
18757                                                if (column.getSourceTable() != null && modelManager.getModel(column.getSourceTable()) instanceof Table) {
18758                                                        Table sourceTable = (Table)modelManager.getModel(column.getSourceTable());
18759                                                        if(sourceTable!=null) {
18760                                                                for(TableColumn tableColumn: sourceTable.getColumns()) {
18761                                                                        if(tableColumn.isStruct()) {
18762                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
18763                                                                                if (names.get(0).equalsIgnoreCase(column.getColumnNameOnly())) {
18764                                                                                        TObjectName columnName = new TObjectName();
18765                                                                                        columnName.setString(tableColumn.getName());
18766                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(unnestTable,
18767                                                                                                        columnName, true);
18768                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18769                                                                                        relation.setEffectType(EffectType.select);
18770                                                                                        relation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
18771                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18772                                                                                        find = true;
18773                                                                                }
18774                                                                        }
18775                                                                }
18776                                                        }
18777                                                }
18778                                                if (!find) {
18779                                                        TObjectName colName = column;
18780                                                        if (table.getAliasClause() != null && table.getAliasClause().getAliasName() != null) {
18781                                                                colName = table.getAliasClause().getAliasName();
18782                                                        }
18783                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, colName, true);
18784                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
18785                                                        columns.add(column);
18786                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18787                                                }
18788                                        }
18789                                }
18790                        }
18791                        return;
18792                }
18793                List<TExpression> expressions = new ArrayList<TExpression>();
18794                TExpressionList values = arrayExpr.getExprList();
18795                if (values == null) {
18796                        expressions.add(arrayExpr);
18797                }
18798                else {
18799                        for(TExpression value: values) {
18800                                expressions.add(value);
18801                        }
18802                }
18803                for (TExpression value : expressions) {
18804                        unnestTable.setCreateTable(true);
18805                        if (value.getExpressionType() == EExpressionType.simple_object_name_t) {
18806                                if (table.getAliasClause() != null) {
18807                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
18808                                                        table.getAliasClause().getAliasName(), true);
18809                                        columnsInExpr visitor = new columnsInExpr();
18810                                        value.inOrderTraverse(visitor);
18811                                        List<TObjectName> columns = visitor.getObjectNames();
18812                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18813                                } else {
18814                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
18815                                        for (int i = 0; i < resultColumnList.size(); i++) {
18816                                                TObjectName firstColumn = new TObjectName();
18817                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
18818                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
18819                                                columnsInExpr visitor = new columnsInExpr();
18820                                                value.inOrderTraverse(visitor);
18821                                                List<TObjectName> columns = visitor.getObjectNames();
18822                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18823                                        }
18824                                }
18825                        } else if (value.getExpressionType() == EExpressionType.function_t) {
18826                                if (table.getAliasClause() != null) {
18827                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
18828                                                        table.getAliasClause().getAliasName(), true);
18829                                        columnsInExpr visitor = new columnsInExpr();
18830                                        value.inOrderTraverse(visitor);
18831                                        List<TParseTreeNode> functions = visitor.getFunctions();
18832                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.select, null);
18833                                } else {
18834                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
18835                                        for (int i = 0; i < resultColumnList.size(); i++) {
18836                                                TObjectName firstColumn = new TObjectName();
18837                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
18838                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
18839                                                columnsInExpr visitor = new columnsInExpr();
18840                                                value.inOrderTraverse(visitor);
18841                                                List<TObjectName> columns = visitor.getObjectNames();
18842                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18843                                        }
18844                                }
18845                        } else if (value.getExpressionType() == EExpressionType.simple_constant_t) {
18846                                if (table.getAliasClause() != null) {
18847                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
18848                                                        table.getAliasClause().getAliasName(), true);
18849                                        columnsInExpr visitor = new columnsInExpr();
18850                                        value.inOrderTraverse(visitor);
18851                                        List<TParseTreeNode> constants = visitor.getConstants();
18852                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
18853                                } else {
18854                                        TObjectName firstColumn = new TObjectName();
18855                                        firstColumn.setString("f0_");
18856                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
18857                                        columnsInExpr visitor = new columnsInExpr();
18858                                        value.inOrderTraverse(visitor);
18859                                        List<TParseTreeNode> constants = visitor.getConstants();
18860                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
18861                                }
18862                        } else if (value.getExpressionType() == EExpressionType.list_t) {
18863                                if (arrayExpr.getTypeName() != null && arrayExpr.getTypeName().getColumnDefList() != null) {
18864                                        for (int i = 0; i < arrayExpr.getTypeName().getColumnDefList().size(); i++) {
18865                                                TColumnDefinition column = arrayExpr.getTypeName().getColumnDefList().getColumn(i);
18866                                                if (column != null && column.getColumnName() != null) {
18867                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
18868                                                                        column.getColumnName(), true);
18869                                                        columnsInExpr visitor = new columnsInExpr();
18870                                                        value.getExprList().getExpression(i).inOrderTraverse(visitor);
18871                                                        List<TParseTreeNode> constants = visitor.getConstants();
18872                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
18873                                                }
18874                                        }
18875                                } else {
18876                                        for (int i = 0; i < value.getExprList().size(); i++) {
18877                                                TObjectName firstColumn = new TObjectName();
18878                                                firstColumn.setString("f" + i + "_");
18879                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
18880                                                columnsInExpr visitor = new columnsInExpr();
18881                                                value.getExprList().getExpression(i).inOrderTraverse(visitor);
18882                                                List<TParseTreeNode> constants = visitor.getConstants();
18883                                                analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
18884                                        }
18885                                }
18886                        } else if (value.getExpressionType() == EExpressionType.subquery_t) {
18887                                analyzeSelectStmt(value.getSubQuery());
18888                                ResultSet resultSet = (ResultSet) modelManager.getModel(value.getSubQuery());
18889                                if (resultSet != null) {
18890                                        for (int i = 0; i < resultSet.getColumns().size(); i++) {
18891                                                TObjectName columnName =  new TObjectName();
18892                                                if(resultSet.getColumns().get(i).getAlias()!=null) {
18893                                                        columnName.setString(resultSet.getColumns().get(i).getAlias());
18894                                                }
18895                                                else {
18896                                                        columnName.setString(getColumnName(resultSet.getColumns().get(i).getName()));
18897                                                }
18898                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
18899                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18900                                                relation.setEffectType(EffectType.select);
18901                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18902                                                relation.addSource(
18903                                                                new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
18904                                        }
18905                                }
18906                        }
18907                }
18908        }
18909
18910        private void analyzePrestoUnnest(TSelectSqlStatement stmt, TTable table) {
18911                List<Table> tables = new ArrayList<Table>();
18912                TTable targetTable = stmt.getTables().getTable(0);
18913                Table stmtTable = modelFactory.createTable(targetTable);
18914                Object sourceTable = null;
18915                if (targetTable.getSubquery() != null) {
18916                        analyzeSelectStmt(targetTable.getSubquery());
18917                        if (targetTable.getSubquery().isCombinedQuery()) {
18918                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(targetTable.getSubquery());
18919                                sourceTable = sourceResultSet;
18920                        } else if (targetTable.getSubquery().getResultColumnList() != null) {
18921                                ResultSet sourceResultSet = (ResultSet) modelManager
18922                                                .getModel(targetTable.getSubquery().getResultColumnList());
18923                                sourceTable = sourceResultSet;
18924                        } else if (targetTable.getSubquery().getValueClause() != null) {
18925                                List<TResultColumnList> rowList = targetTable.getSubquery().getValueClause().getRows();
18926                                if (rowList != null && rowList.size() > 0) {
18927                                        Table valuesTable = modelFactory.createTableByName("Values-Table", true);
18928                                        int columnCount = rowList.get(0).size();
18929                                        for (int j = 1; j <= columnCount; j++) {
18930                                                TObjectName columnName = new TObjectName();
18931                                                TResultColumn columnObject = rowList.get(0).getResultColumn(j - 1);
18932                                                if (columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t) {
18933                                                        columnName.setString(columnObject.getExpr().getLeftOperand().toString());
18934                                                } else {
18935                                                        columnName.setString(columnObject.getExpr().toString());
18936                                                }
18937                                                modelFactory.createTableColumn(valuesTable, columnName, true);
18938                                        }
18939                                        valuesTable.setCreateTable(true);
18940                                        valuesTable.setSubType(SubType.values_table);
18941                                        sourceTable = valuesTable;
18942                                }
18943                        }
18944                }
18945                tables.add(stmtTable);
18946                Table unnestTable = modelFactory.createTable(table);
18947                unnestTable.setSubType(SubType.unnest);
18948                tables.add(unnestTable);
18949                if (table.getAliasClause() != null && table.getAliasClause().getColumns() != null
18950                                && table.getUnnestClause().getColumns() != null) {
18951                        int unnestTableSize = table.getUnnestClause().getColumns().size();
18952                        for (int i = 0; i < table.getAliasClause().getColumns().size(); i++) {
18953                                TableColumn sourceColumn = null;
18954                                if (unnestTableSize > i) {
18955                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
18956                                                        table.getUnnestClause().getColumns().getObjectName(i), true);
18957                                        if (sourceTable instanceof Table) {
18958                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18959                                                relation.setEffectType(EffectType.select);
18960                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
18961                                                relation.addSource(
18962                                                                new TableColumnRelationshipElement(((Table) sourceTable).getColumns().get(i)));
18963                                        } else if (sourceTable instanceof ResultSet) {
18964                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18965                                                relation.setEffectType(EffectType.select);
18966                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
18967                                                relation.addSource(
18968                                                                new ResultColumnRelationshipElement(((ResultSet) sourceTable).getColumns().get(i)));
18969                                        }
18970                                } else {
18971                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
18972                                                        table.getUnnestClause().getColumns().getObjectName(unnestTableSize - 1), true);
18973                                }
18974                                TableColumn targetColumn = modelFactory.createTableColumn(unnestTable,
18975                                                table.getAliasClause().getColumns().getObjectName(i), true);
18976                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18977                                relation.setEffectType(EffectType.select);
18978                                relation.setTarget(new TableColumnRelationshipElement(targetColumn));
18979                                relation.addSource(new TableColumnRelationshipElement(sourceColumn));
18980                        }
18981                }
18982
18983                ResultSet resultSet = modelFactory.createResultSet(stmt,
18984                                isTopResultSet(stmt) && isShowTopSelectResultSet());
18985                TResultColumnList columnList = stmt.getResultColumnList();
18986                for (int i = 0; i < columnList.size(); i++) {
18987                        TResultColumn column = columnList.getResultColumn(i);
18988                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
18989                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
18990                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
18991                                if (columnObject.getFieldAttr() != null) {
18992                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
18993                                                for (int k = 0; k < tables.size(); k++) {
18994                                                        Table tableItem = tables.get(k);
18995                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
18996                                                                resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
18997                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18998                                                                relation.setEffectType(EffectType.select);
18999                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
19000                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19001                                                        }
19002                                                }
19003                                        } else {
19004                                                boolean match = false;
19005                                                for (int k = 0; k < tables.size(); k++) {
19006                                                        Table tableItem = tables.get(k);
19007                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19008                                                                if (getColumnName(columnObject.getFieldAttr())
19009                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19010                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19011                                                                        relation.setEffectType(EffectType.select);
19012                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19013                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19014                                                                        match = true;
19015                                                                }
19016                                                        }
19017                                                }
19018                                                if (!match) {
19019                                                        TableColumn tableColumn = modelFactory.createTableColumn(stmtTable,
19020                                                                        columnObject.getFieldAttr(), false);
19021                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19022                                                        relation.setEffectType(EffectType.select);
19023                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19024                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19025                                                }
19026                                        }
19027                                }
19028                        }
19029                }
19030        }
19031
19032        private void analyzeLateralView(TSelectSqlStatement stmt, TTable table, ArrayList<TLateralView> lateralViews) {
19033                List<Object> tables = new ArrayList<Object>();
19034                Object stmtTable = null;
19035                if(table.getSubquery()!=null) {
19036                        stmtTable = modelFactory.createQueryTable(table);
19037                        tables.add(stmtTable);
19038                }
19039                else {
19040                        stmtTable = modelFactory.createTable(table);
19041                        tables.add(stmtTable);
19042                }
19043                for (int i = 0; i < lateralViews.size(); i++) {
19044                        TLateralView lateralView = lateralViews.get(i);
19045                        TFunctionCall functionCall = lateralView.getUdtf();
19046                        List<TExpression> expressions = new ArrayList<TExpression>();
19047                        if (functionCall == null) {
19048                                continue;
19049                        }
19050                        Function function = modelFactory.createFunction(functionCall);
19051                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
19052                                        ((TFunctionCall) functionCall).getFunctionName());
19053
19054                        Table lateralTable = null;
19055                        if (lateralView.getTableAlias() != null) {
19056                                lateralTable = modelFactory.createTableByName(lateralView.getTableAlias().getAliasName(), true);
19057                        } else {
19058                                lateralTable = modelFactory.createTableByName(functionCall.toString(), true);
19059                        }
19060
19061                        for (int j = 0; j < lateralView.getColumnAliasList().size(); j++) {
19062                                TObjectName viewColumn = lateralView.getColumnAliasList().getObjectName(j);
19063                                TableColumn tableColumn = modelFactory.createTableColumn(lateralTable, viewColumn, true);
19064
19065                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19066                                relation.setEffectType(EffectType.select);
19067                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
19068                                relation.addSource(new ResultColumnRelationshipElement(column));
19069                        }
19070
19071                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
19072                        for (int j = 0; j < expressions.size(); j++) {
19073                                columnsInExpr visitor = new columnsInExpr();
19074                                expressions.get(j).inOrderTraverse(visitor);
19075                                List<TObjectName> objectNames = visitor.getObjectNames();
19076                                if (objectNames == null) {
19077                                        continue;
19078                                }
19079                                for (TObjectName columnName : objectNames) {
19080                                        boolean match = false;
19081                                        for (int k = 0; k < tables.size(); k++) {
19082                                                Object item = tables.get(k);
19083                                                if(item instanceof Table) {
19084                                                        Table tableItem = (Table)item;
19085                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19086                                                                if (getColumnName(columnName)
19087                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19088                                                                        match = true;
19089                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19090                                                                        relation.setEffectType(EffectType.select);
19091                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19092                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19093                                                                }
19094                                                        }
19095                                                }
19096                                                else {
19097                                                        ResultSet tableItem = (ResultSet)item;
19098                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
19099                                                                if (getColumnName(columnName)
19100                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19101                                                                        match = true;
19102                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19103                                                                        relation.setEffectType(EffectType.select);
19104                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19105                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19106                                                                }
19107                                                        }
19108                                                }
19109                                        }
19110                                        if (!match) {
19111                                                if(stmtTable instanceof Table) {
19112                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable, columnName, false);
19113                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19114                                                        relation.setEffectType(EffectType.select);
19115                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19116                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19117                                                }
19118                                                else {
19119                                                        ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable, columnName, false);
19120                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19121                                                        relation.setEffectType(EffectType.select);
19122                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19123                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19124                                                }
19125                                        }
19126                                }
19127                                List<TParseTreeNode> constants = visitor.getConstants();
19128                                if (!constants.isEmpty()) {
19129                                        if (option.isShowConstantTable()) {
19130                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
19131                                                for (TParseTreeNode constant : constants) {
19132                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19133                                                        relation.setEffectType(EffectType.select);
19134                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19135                                                        if (constant instanceof TConstant) {
19136                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
19137                                                                                (TConstant) constant);
19138                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
19139                                                        } else if (constant instanceof TObjectName) {
19140                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
19141                                                                                (TObjectName) constant, false);
19142                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
19143                                                        }
19144                                                }
19145                                        }
19146                                }
19147                                
19148                                List<TParseTreeNode> functions = visitor.getFunctions();
19149                                if (functions != null && !functions.isEmpty()) {
19150                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
19151                                }
19152                        }
19153                        tables.add(lateralTable);
19154                }
19155
19156                ResultSet resultSet = modelFactory.createResultSet(stmt,
19157                                isTopResultSet(stmt) && isShowTopSelectResultSet());
19158                TResultColumnList columnList = stmt.getResultColumnList();
19159                for (int i = 0; i < columnList.size(); i++) {
19160                        TResultColumn column = columnList.getResultColumn(i);
19161                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
19162                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
19163                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
19164                                if (columnObject.getFieldAttr() != null) {
19165                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
19166                                                for (int k = 0; k < tables.size(); k++) {
19167                                                        Object item = tables.get(k);
19168                                                        if(item instanceof Table) {
19169                                                                Table tableItem = (Table)item;
19170                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
19171                                                                        resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
19172                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19173                                                                        relation.setEffectType(EffectType.select);
19174                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
19175                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19176                                                                }
19177                                                        }
19178                                                        else {
19179                                                                ResultSet tableItem = (ResultSet)item;
19180                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
19181                                                                        TObjectName linkColumn = new TObjectName();
19182                                                                        linkColumn.setString(tableColumn.getName());
19183                                                                        resultColumn.bindStarLinkColumn(linkColumn);
19184                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19185                                                                        relation.setEffectType(EffectType.select);
19186                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, linkColumn));
19187                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19188                                                                }
19189                                                        }
19190                                                }
19191                                        } else {
19192                                                boolean match = false;
19193                                                for (int k = 0; k < tables.size(); k++) {
19194                                                        Object item = tables.get(k);
19195                                                        if(item instanceof Table) {
19196                                                                Table tableItem = (Table)item;
19197                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
19198                                                                        if (getColumnName(columnObject.getFieldAttr())
19199                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19200                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19201                                                                                relation.setEffectType(EffectType.select);
19202                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19203                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19204                                                                                match = true;
19205                                                                        }
19206                                                                }
19207                                                        }
19208                                                        else {
19209                                                                ResultSet tableItem = (ResultSet)item;
19210                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
19211                                                                        if (getColumnName(columnObject.getFieldAttr())
19212                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19213                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19214                                                                                relation.setEffectType(EffectType.select);
19215                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19216                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19217                                                                                match = true;
19218                                                                        }
19219                                                                }
19220                                                        }
19221                                                }
19222                                                if (!match) {
19223                                                        if(stmtTable instanceof Table) {
19224                                                                TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable,
19225                                                                                columnObject.getFieldAttr(), false);
19226                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19227                                                                relation.setEffectType(EffectType.select);
19228                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19229                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19230                                                        }
19231                                                        else {
19232                                                                ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable,
19233                                                                                columnObject.getFieldAttr(), false);
19234                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19235                                                                relation.setEffectType(EffectType.select);
19236                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19237                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19238                                                        }
19239                                                }
19240                                        }
19241                                }
19242                                else if (columnObject.getExpr() != null
19243                                                && columnObject.getExpr().getExpressionType() == EExpressionType.function_t) {
19244                                        analyzeResultColumn(column, EffectType.select);
19245                                        for (int k = 0; k < tables.size(); k++) {
19246                                                Object item = tables.get(k);
19247                                                if (item instanceof Table) {
19248                                                        Table tableItem = (Table) item;
19249                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19250                                                                if (getColumnName(resultColumn.getName()).equals(
19251                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19252                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19253                                                                        relation.setEffectType(EffectType.select);
19254                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19255                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19256                                                                }
19257                                                        }
19258                                                } else {
19259                                                        ResultSet tableItem = (ResultSet) item;
19260                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
19261                                                                if (getColumnName(resultColumn.getName()).equals(
19262                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19263                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19264                                                                        relation.setEffectType(EffectType.select);
19265                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19266                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19267                                                                }
19268                                                        }
19269                                                }
19270                                        }
19271                                }
19272                        }
19273                }
19274        }
19275
19276        private boolean isFromFunction(TObjectName object) {
19277
19278                Stack<TParseTreeNode> nodes = object.getStartToken().getNodesStartFromThisToken();
19279                if (nodes != null) {
19280                        for (int i = 0; i < nodes.size(); i++) {
19281                                if (nodes.get(i) instanceof TFunctionCall) {
19282                                        return true;
19283                                }
19284                        }
19285                }
19286                return false;
19287        }
19288
19289        private TResultColumnList getResultColumnList(TSelectSqlStatement stmt) {
19290                // Iterative DFS (left-first) to find the first non-combined query's result column list.
19291                // Avoids StackOverflow with deeply nested UNION trees.
19292                Deque<TSelectSqlStatement> stack = new ArrayDeque<>();
19293                stack.push(stmt);
19294                while (!stack.isEmpty()) {
19295                        TSelectSqlStatement current = stack.pop();
19296                        if (current.isCombinedQuery()) {
19297                                // Push right first so left is processed first (stack is LIFO)
19298                                if (current.getRightStmt() != null) stack.push(current.getRightStmt());
19299                                if (current.getLeftStmt() != null) stack.push(current.getLeftStmt());
19300                        } else {
19301                                if (current.getResultColumnList() != null) {
19302                                        return current.getResultColumnList();
19303                                }
19304                        }
19305                }
19306                return null;
19307        }
19308
19309        private void createPseudoImpactRelation(TCustomSqlStatement stmt, ResultSet resultSetModel, EffectType effectType) {
19310                if (stmt.getTables() != null) {
19311                        for (int i = 0; i < stmt.getTables().size(); i++) {
19312                                TTable table = stmt.getTables().getTable(i);
19313                                if (modelManager.getModel(table) instanceof ResultSet) {
19314                                        ResultSet tableModel = (ResultSet) modelManager.getModel(table);
19315                                        if (tableModel != resultSetModel && !tableModel.getRelationRows().getHoldRelations().isEmpty()) {
19316                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19317                                                impactRelation.setEffectType(effectType);
19318                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19319                                                                tableModel.getRelationRows()));
19320                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19321                                                                resultSetModel.getRelationRows()));
19322                                        }
19323                                } else if (modelManager.getModel(table) instanceof Table) {
19324                                        Table tableModel = (Table) modelManager.getModel(table);
19325                                        if (!tableModel.getRelationRows().getHoldRelations().isEmpty()) {
19326                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19327                                                impactRelation.setEffectType(effectType);
19328                                                impactRelation.addSource(
19329                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
19330                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19331                                                                resultSetModel.getRelationRows()));
19332                                        }
19333                                }
19334                        }
19335                }
19336        }
19337
19338        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
19339                        EffectType effectType) {
19340                for (int i = 0; i < functions.size(); i++) {
19341                        TParseTreeNode functionCall = functions.get(i);
19342                        if (functionCall instanceof TFunctionCall) {
19343                                String functionName = DlineageUtil.getIdentifierNormalTableName(
19344                                                DlineageUtil.getFunctionNameWithArgNum((TFunctionCall) functionCall));
19345                                Procedure procedure = modelManager.getProcedureByName(functionName);
19346                                if (procedure != null) {
19347                                        String procedureParent = getProcedureParentName(stmtStack.peek());
19348                                        if (procedureParent != null) {
19349                                                Procedure caller = modelManager
19350                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19351                                                if (caller != null) {
19352                                                        CallRelationship callRelation = modelFactory.createCallRelation();
19353                                                        callRelation.setCallObject(functionCall);
19354                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
19355                                                        callRelation.addSource(new ProcedureRelationshipElement(procedure));
19356                                                        if (isBuiltInFunctionName(((TFunctionCall)functionCall).getFunctionName())||isKeyword(((TFunctionCall)functionCall).getFunctionName())) {
19357                                                                callRelation.setBuiltIn(true);
19358                                                        }
19359                                                }
19360                                        }
19361                                        if (procedure.getProcedureObject() instanceof TCreateFunctionStmt) {
19362                                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt)procedure.getProcedureObject();
19363                                                TTypeName dataType = createFunction.getReturnDataType();
19364                                                if (dataType!=null && dataType.getTypeOfList() != null && dataType.getTypeOfList().getColumnDefList() != null) {
19365                                                        Object modelObject = modelManager.getModel(gspObject);
19366                                                        if(modelObject instanceof ResultColumn) {
19367                                                                ResultColumn resultColumn = (ResultColumn)modelObject;
19368                                                                ResultSet resultSet = resultColumn.getResultSet();
19369                                                                for (int j = 0; j < dataType.getTypeOfList().getColumnDefList().size(); j++) {
19370                                                                        TObjectName columnName = new TObjectName();
19371                                                                        if( dataType.getDataType() == EDataType.array_t) {
19372//                                                                              columnName.setString(resultColumn.getName() + ".array."
19373//                                                                                              + dataType.getTypeOfList().getColumnDefList().getColumn(j)
19374//                                                                                                              .getColumnName().getColumnNameOnly());
19375                                                                                columnName.setString(resultColumn.getName() + "."
19376                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
19377                                                                                                .getColumnName().getColumnNameOnly());
19378                                                                        }
19379                                                                        else {
19380                                                                                columnName.setString(resultColumn.getName() + "."
19381                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
19382                                                                                                                .getColumnName().getColumnNameOnly());
19383                                                                        }
19384                                                                        ResultColumn sturctColumn = modelFactory.createResultColumn(resultSet, columnName,
19385                                                                                        true);
19386                                                                        sturctColumn.setStruct(true);
19387                                                                        Function sourceFunction = (Function)createFunction(functionCall);
19388                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19389                                                                        relation.setEffectType(effectType);
19390                                                                        relation.setTarget(new ResultColumnRelationshipElement(sturctColumn));
19391                                                                        if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
19392                                                                                for (ResultColumn column : sourceFunction.getColumns()) {
19393                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
19394                                                                                }
19395                                                                        }
19396                                                                }
19397                                                                resultSet.getColumns().remove(resultColumn);
19398                                                        }
19399                                                        return;
19400                                                }
19401                                        }
19402                                }
19403                        }
19404                        
19405                        if(gspObject instanceof TResultColumn) {
19406                                TResultColumn resultColumn = (TResultColumn)gspObject;
19407                                if(resultColumn.getAliasClause()!=null && resultColumn.getAliasClause().getColumns()!=null) {
19408                                        for(TObjectName columnName: resultColumn.getAliasClause().getColumns()) {
19409                                                analyzeFunctionDataFlowRelation(columnName, Arrays.asList(functionCall), effectType, null);
19410                                        }
19411                                        return;
19412                                }
19413                        }
19414                        analyzeFunctionDataFlowRelation(gspObject, Arrays.asList(functionCall), effectType, null);
19415                }
19416        }
19417
19418        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
19419                        EffectType effectType, Process process) {
19420
19421                Object modelObject = modelManager.getModel(gspObject);
19422                if (modelObject == null) {
19423                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
19424                                modelObject = gspObject;
19425                        }
19426                }
19427
19428                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19429                relation.setEffectType(effectType);
19430                relation.setProcess(process);
19431
19432                if (modelObject instanceof ResultColumn) {
19433                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
19434
19435                } else if (modelObject instanceof TableColumn) {
19436                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
19437
19438                } else {
19439                        throw new UnsupportedOperationException();
19440                }
19441
19442                for (int i = 0; i < functions.size(); i++) {
19443                        TParseTreeNode functionCall = functions.get(i);
19444
19445                        if (functionCall instanceof TFunctionCall) {
19446                                TFunctionCall call = (TFunctionCall) functionCall;
19447                                Procedure callee = modelManager.getProcedureByName(
19448                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
19449                                if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(call))) {
19450                                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(call)));
19451                                        callee = modelManager.getProcedureByName(
19452                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
19453                                }
19454
19455                                if (callee != null) {
19456                                        String procedureParent = getProcedureParentName(stmtStack.peek());
19457                                        if (procedureParent != null) {
19458                                                Procedure caller = modelManager
19459                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19460                                                if (caller != null) {
19461                                                        CallRelationship callRelation = modelFactory.createCallRelation();
19462                                                        callRelation.setCallObject(functionCall);
19463                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
19464                                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
19465                                                        if (isBuiltInFunctionName(call.getFunctionName()) || isKeyword(call.getFunctionName())) {
19466                                                                callRelation.setBuiltIn(true);
19467                                                        }
19468                                                }
19469                                        }
19470                                        if (callee.getArguments() != null) {
19471                                                for (int j = 0; j < callee.getArguments().size(); j++) {
19472                                                        Argument argument = callee.getArguments().get(j);
19473                                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
19474                                                        if(variable!=null) {
19475                                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
19476                                                                        Transform transform = new Transform();
19477                                                                        transform.setType(Transform.FUNCTION);
19478                                                                        transform.setCode(call);
19479                                                                        variable.getColumns().get(0).setTransform(transform);
19480                                                                }
19481                                                                Process callProcess = modelFactory.createProcess(call);
19482                                                                variable.addProcess(callProcess);
19483                                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), call, j, callProcess);
19484                                                        }
19485                                                }
19486                                        }
19487                                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
19488                                                        .getIdentifierNormalTableName(call.getFunctionName().toString()));
19489                                        if (functionTableModelObjs != null) {
19490                                                modelManager.bindModel(call, functionTableModelObjs.iterator().next());
19491                                                for (Object functionTableModelObj : functionTableModelObjs) {
19492                                                        if (functionTableModelObj instanceof ResultSet) {
19493                                                                ResultSet resultSet = (ResultSet) functionTableModelObj;
19494                                                                for (ResultColumn column : resultSet.getColumns()) {
19495                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
19496                                                                }
19497                                                        }
19498                                                }
19499                                        }
19500                                        continue;
19501                                }
19502                        }
19503
19504
19505                        Object functionModel = createFunction(functionCall);
19506                        if (functionModel instanceof Function) {
19507                                Function sourceFunction = (Function)functionModel;
19508                                if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
19509                                        for (ResultColumn column : sourceFunction.getColumns()) {
19510                                                relation.addSource(new ResultColumnRelationshipElement(column));
19511                                        }
19512                                }
19513                                else if (functionCall instanceof TFunctionCall) {
19514                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
19515                                                        .getModel(((TFunctionCall) functionCall).getFunctionName())));
19516                                } else if (functionCall instanceof TCaseExpression) {
19517                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
19518                                                        .getModel(((TCaseExpression) functionCall).getWhenClauseItemList())));
19519                                }
19520                                
19521                                if (sourceFunction != null && !sourceFunction.getRelationRows().getHoldRelations().isEmpty()) {
19522                                        boolean find = false;
19523                                        if (modelObject instanceof ResultColumn) {
19524                                                ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
19525                                                if(targetRelationRows.hasRelation()) {
19526                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19527                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
19528                                                                        find = true;
19529                                                                        break;
19530                                                                }
19531                                                        }
19532                                                }
19533                                        }
19534                                        else if (modelObject instanceof TableColumn) {
19535                                                TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
19536                                                if(targetRelationRows.hasRelation()) {
19537                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19538                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
19539                                                                        find = true;
19540                                                                        break;
19541                                                                }
19542                                                        }
19543                                                }
19544                                        }
19545                                        
19546                                        if (!find) {
19547                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19548                                                impactRelation.setEffectType(EffectType.select);
19549                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19550                                                                sourceFunction.getRelationRows()));
19551                                                if (modelObject instanceof ResultColumn) {
19552                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19553                                                                        ((ResultColumn) modelObject).getResultSet().getRelationRows()));
19554                                                } else if (modelObject instanceof TableColumn) {
19555                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
19556                                                                        ((TableColumn) modelObject).getTable().getRelationRows()));
19557                                                }
19558                                        }
19559                                }
19560                        } else if (functionModel instanceof Table) {
19561                                TFunctionCall call = (TFunctionCall) functionCall;
19562                                String functionName = call.getFunctionName().toString();
19563                                boolean flag = false;
19564                                if (functionName.indexOf(".") != -1) {
19565                                        String columnName = functionName.substring(functionName.indexOf(".") + 1);
19566                                        for (TableColumn tableColumn : ((Table) functionModel).getColumns()) {
19567                                                if (getColumnName(tableColumn.getName()).equalsIgnoreCase(columnName)) {
19568                                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
19569                                                        relation.addSource(element);
19570                                                        flag = true;
19571                                                        break;
19572                                                }
19573                                        }
19574                                }
19575
19576                                if (!flag) {
19577                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
19578                                                        ((TFunctionCall) functionCall));
19579                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
19580                                        relation.addSource(element);
19581                                }
19582                        }
19583                }
19584
19585        }
19586
19587        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
19588                        EffectType effectType) {
19589                analyzeSubqueryDataFlowRelation(gspObject, subquerys, effectType, null);
19590        }
19591
19592        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
19593                        EffectType effectType, Process process) {
19594
19595                Object modelObject = modelManager.getModel(gspObject);
19596                if (modelObject == null) {
19597                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
19598                                modelObject = gspObject;
19599                        }
19600                }
19601
19602                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19603                relation.setEffectType(effectType);
19604                relation.setProcess(process);
19605
19606                if (modelObject instanceof ResultColumn) {
19607                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
19608
19609                } else if (modelObject instanceof TableColumn) {
19610                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
19611
19612                } else {
19613                        throw new UnsupportedOperationException();
19614                }
19615
19616                for (int i = 0; i < subquerys.size(); i++) {
19617                        TSelectSqlStatement subquery = subquerys.get(i);
19618                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
19619                        if (resultSetModel != null && resultSetModel.getColumns() != null) {
19620                                for (ResultColumn column : resultSetModel.getColumns()) {
19621                                        relation.addSource(new ResultColumnRelationshipElement(column));
19622                                }
19623                        }
19624                        
19625                        if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
19626                                boolean find = false;
19627                                if (modelObject instanceof ResultColumn) {
19628                                        ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
19629                                        if(targetRelationRows.hasRelation()) {
19630                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19631                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
19632                                                                find = true;
19633                                                                break;
19634                                                        }
19635                                                }
19636                                        }
19637                                }
19638                                else if (modelObject instanceof TableColumn) {
19639                                        TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
19640                                        if(targetRelationRows.hasRelation()) {
19641                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19642                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
19643                                                                find = true;
19644                                                                break;
19645                                                        }
19646                                                }
19647                                        }
19648                                }
19649                                
19650                                if (!find) {
19651                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19652                                        impactRelation.setEffectType(EffectType.select);
19653                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19654                                                        resultSetModel.getRelationRows()));
19655                                        if (modelObject instanceof ResultColumn) {
19656                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19657                                                                ((ResultColumn) modelObject).getResultSet().getRelationRows()));
19658                                        } else if (modelObject instanceof TableColumn) {
19659                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
19660                                                                ((TableColumn) modelObject).getTable().getRelationRows()));
19661                                        }
19662                                }
19663                        }
19664                }
19665
19666        }
19667
19668        private Object createFunction(TParseTreeNode functionCall) {
19669                if (functionCall instanceof TFunctionCall) {
19670                        TFunctionCall functionObj = (TFunctionCall) functionCall;
19671                        if (!isBuiltInFunctionName(functionObj.getFunctionName())) {
19672                                TCustomSqlStatement stmt = stmtStack.peek();
19673                                String procedureParent = getProcedureParentName(stmt);
19674                                if (procedureParent != null) {
19675                                        Procedure procedureCallee = modelManager.getProcedureByName(DlineageUtil
19676                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionObj)));
19677                                        if (procedureCallee != null) {
19678                                                if (procedureParent != null) {
19679                                                        Procedure caller = modelManager
19680                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19681                                                        if (caller != null) {
19682                                                                CallRelationship callRelation = modelFactory.createCallRelation();
19683                                                                callRelation.setCallObject(functionCall);
19684                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
19685                                                                callRelation.addSource(new ProcedureRelationshipElement(procedureCallee));
19686                                                                if(isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())){
19687                                                                        callRelation.setBuiltIn(true);
19688                                                                }
19689                                                        }
19690                                                }
19691                                                if (procedureCallee.getArguments() != null) {
19692                                                        for (int j = 0; j < procedureCallee.getArguments().size(); j++) {
19693                                                                Argument argument = procedureCallee.getArguments().get(j);
19694                                                                Variable variable = modelFactory.createVariable(procedureCallee, argument.getName(), false);
19695                                                                if(variable!=null) {
19696                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
19697                                                                                Transform transform = new Transform();
19698                                                                                transform.setType(Transform.FUNCTION);
19699                                                                                transform.setCode(functionObj);
19700                                                                                variable.getColumns().get(0).setTransform(transform);
19701                                                                        }
19702                                                                        Process process = modelFactory.createProcess(functionObj);
19703                                                                        variable.addProcess(process);
19704                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionObj, j, process);
19705                                                                }
19706                                                        }
19707                                                }
19708                                        } else {
19709                                                TFunctionCall call = (TFunctionCall)functionCall;
19710                                                String functionName = call.getFunctionName().toString();
19711                                                if (functionName.indexOf(".") != -1) {
19712                                                        Table functionTable = modelManager
19713                                                                        .getTableByName(functionName.substring(0, functionName.indexOf(".")));
19714                                                        if (functionTable != null) {
19715                                                                String columnName = functionName.substring(functionName.indexOf(".") + 1);
19716                                                                for (TableColumn tableColumn : functionTable.getColumns()) {
19717                                                                        if (getColumnName(tableColumn.getName()).equalsIgnoreCase(columnName)) {
19718                                                                                return functionTable;
19719                                                                        }
19720                                                                }
19721                                                        }
19722                                                }
19723                                                Function function = modelFactory.createFunction(call);
19724                                                if (procedureParent != null) {
19725                                                        Procedure caller = modelManager
19726                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19727                                                        if (caller != null) {
19728                                                                CallRelationship callRelation = modelFactory.createCallRelation();
19729                                                                callRelation.setCallObject(functionCall);
19730                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
19731                                                                callRelation.addSource(new FunctionRelationshipElement(function));
19732                                                                if(isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())){
19733                                                                        callRelation.setBuiltIn(true);
19734                                                                }
19735                                                        }
19736                                                }
19737                                        }
19738                                }
19739                        } else if (isConstantFunction(functionObj.getFunctionName())
19740                                        && (functionObj.getArgs() == null || functionObj.getArgs().size() == 0)) {
19741                                if (option.isShowConstantTable()) {
19742                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
19743                                        modelFactory.createTableColumn(constantTable, functionObj);
19744                                        return constantTable;
19745                                } else {
19746                                        return null;
19747                                }
19748                        }  
19749                        
19750                        if (functionObj.getFunctionType() == EFunctionType.struct_t) {
19751                                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
19752                                if(functionObj instanceof TTableFunction) {
19753                                        TTableFunction tableFunction = (TTableFunction) functionObj;
19754                                        if (tableFunction.getFieldValues() != null) {
19755                                                for (int i = 0; i < tableFunction.getFieldValues().size(); i++) {
19756                                                        TResultColumn resultColumn = tableFunction.getFieldValues().getResultColumn(i);
19757                                                        if (resultColumn.getAliasClause() != null) {
19758                                                                ResultColumn column = modelFactory.createFunctionResultColumn(function,
19759                                                                                resultColumn.getAliasClause().getAliasName());
19760                                                                columnsInExpr visitor = new columnsInExpr();
19761                                                                resultColumn.getExpr().inOrderTraverse(visitor);
19762                                                                List<TObjectName> objectNames = visitor.getObjectNames();
19763                                                                List<TParseTreeNode> functions = visitor.getFunctions();
19764                                                                if (functions != null && !functions.isEmpty()) {
19765                                                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
19766                                                                }
19767                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
19768                                                                if (subquerys != null && !subquerys.isEmpty()) {
19769                                                                        analyzeSubqueryDataFlowRelation(column, subquerys, EffectType.function);
19770                                                                }
19771                                                                analyzeDataFlowRelation(column, objectNames, EffectType.function, functions);
19772                                                                List<TParseTreeNode> constants = visitor.getConstants();
19773                                                                analyzeConstantDataFlowRelation(column, constants, EffectType.function, functions);
19774                                                        } else if (resultColumn.getFieldAttr() != null) {
19775                                                                ResultColumn column = modelFactory.createFunctionResultColumn(function,
19776                                                                                resultColumn.getFieldAttr());
19777                                                                analyzeDataFlowRelation(column, Arrays.asList(resultColumn.getFieldAttr()), EffectType.function, null);
19778                                                        } else if (resultColumn.getExpr() != null) {
19779                                                                if (resultColumn.getExpr().getFunctionCall() != null) {
19780                                                                        Function resultColumnFunction = (Function) createFunction(
19781                                                                                        resultColumn.getExpr().getFunctionCall());
19782                                                                        String functionName = getResultSetName(function);
19783                                                                        for (int j = 0; j < resultColumnFunction.getColumns().size(); j++) {
19784                                                                                TObjectName columnName = new TObjectName();
19785                                                                                if (resultColumn.getAliasClause() != null) {
19786                                                                                        columnName.setString(resultColumn.getAliasClause() + "." + getColumnNameOnly(
19787                                                                                                        resultColumnFunction.getColumns().get(j).getName()));
19788                                                                                } else {
19789                                                                                        columnName.setString(functionName + "." + getColumnNameOnly(
19790                                                                                                        resultColumnFunction.getColumns().get(j).getName()));
19791                                                                                }
19792                                                                                ResultColumn functionResultColumn = modelFactory.createResultColumn(function,
19793                                                                                                columnName);
19794                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
19795                                                                                relationship.setTarget(new ResultColumnRelationshipElement(functionResultColumn));
19796                                                                                relationship.addSource(new ResultColumnRelationshipElement(
19797                                                                                                resultColumnFunction.getColumns().get(j)));
19798                                                                        }
19799                                                                }
19800                                                                else if (resultColumn.getExpr().getCaseExpression() != null) {
19801                                                                        function = modelFactory.createFunction(resultColumn.getExpr().getCaseExpression());
19802                                                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
19803                                                                                        ((TCaseExpression) resultColumn.getExpr().getCaseExpression()).getWhenClauseItemList());
19804                                                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
19805                                                                }
19806                                                        }
19807                                                }
19808                                                return function;
19809                                        }
19810                                }
19811                                else if(functionObj instanceof TFunctionCall) {
19812                                        
19813                                }
19814                        }
19815                        
19816                        if (functionObj.getFunctionType() == EFunctionType.array_t || functionObj.getFunctionType() == EFunctionType.array_agg_t) {
19817                                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
19818                                if(functionObj.getArgs()!=null) {
19819                                        if(functionObj.getArgs().getExpression(0).getSubQuery()!=null) {
19820                                                TSelectSqlStatement stmt = functionObj.getArgs().getExpression(0).getSubQuery();
19821                                                analyzeSelectStmt(stmt);
19822                                                ResultSet resultset = (ResultSet) modelManager.getModel(stmt);
19823                                                for (int i = 0; i < resultset.getColumns().size(); i++) {
19824                                                        ResultColumn sourceColumn = resultset.getColumns().get(i);
19825                                                        TObjectName columnName = new TObjectName();
19826                                                        columnName.setString(sourceColumn.getName());
19827                                                        ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function,
19828                                                                        columnName);
19829                                                        DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
19830                                                        relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
19831                                                        relationship.addSource(
19832                                                                        new ResultColumnRelationshipElement(sourceColumn));
19833                                                }
19834                                                return function;
19835                                        }
19836                                        else if (functionObj.getArgs().getExpression(0).getExpressionType() == EExpressionType.function_t) {
19837                                                Object functionTableModelObj = createFunction(functionObj.getArgs().getExpression(0).getFunctionCall());
19838                                                if (functionTableModelObj instanceof ResultSet) {
19839                                                        ResultSet resultset = (ResultSet) functionTableModelObj;
19840                                                        for (int i = 0; i < resultset.getColumns().size(); i++) {
19841                                                                ResultColumn sourceColumn = resultset.getColumns().get(i);
19842                                                                TObjectName columnName = new TObjectName();
19843                                                                columnName.setString(sourceColumn.getName());
19844                                                                ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function, columnName);
19845                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
19846                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
19847                                                                relationship.addSource(new ResultColumnRelationshipElement(sourceColumn));
19848                                                        }
19849                                                        return function;
19850                                                }
19851                                        }
19852                                }
19853                        }
19854                        
19855                        Function function = modelFactory.createFunction((TFunctionCall) functionCall);
19856                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
19857                                        ((TFunctionCall) functionCall).getFunctionName());
19858                        if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
19859                                // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
19860                                // COUNT特殊处理，不和参数关联
19861                                if (option.isShowCountTableColumn()) {
19862                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
19863                                }
19864                        } else {
19865                                boolean isCustomFunction = analyzeCustomFunctionCall((TFunctionCall)functionCall);
19866//                              if(!isCustomFunction) 
19867                                {
19868                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
19869                                }
19870                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
19871                                if(functionTableModelObjs!=null) {
19872                                        for(Object functionTableModelObj: functionTableModelObjs) {
19873                                                if (functionTableModelObj instanceof ResultSet) {
19874                                                        ResultSet functionTableModel = (ResultSet) functionTableModelObj;
19875                                                        if (functionTableModel.getColumns() != null) {
19876                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
19877                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19878                                                                        relation.setEffectType(EffectType.select);
19879                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19880                                                                        relation.addSource(new ResultColumnRelationshipElement(
19881                                                                                        functionTableModel.getColumns().get(j)));
19882                                                                }
19883                                                        }
19884                                                }
19885                                        }
19886                                }
19887                        }
19888                        return function;
19889                } else if (functionCall instanceof TCaseExpression) {
19890                        Function function = modelFactory.createFunction((TCaseExpression) functionCall);
19891                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
19892                                        ((TCaseExpression) functionCall).getWhenClauseItemList());
19893                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
19894                        return function;
19895                } else if (functionCall instanceof TObjectName) {
19896                        Function function = modelFactory.createFunction((TObjectName) functionCall);
19897                        TObjectName columnName = new TObjectName();
19898                        columnName.setString(function.getFunctionName());
19899                        ResultColumn column = modelFactory.createResultColumn(function,
19900                                        columnName);
19901                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
19902                        return function;
19903                }
19904                return null;
19905        }
19906
19907        protected String getIdentifiedFunctionName(Function function) {
19908                return DlineageUtil.getIdentifierNormalFunctionName(function.getFunctionName());
19909        }
19910        
19911        private boolean isConstantFunction(TObjectName functionName) {
19912                boolean result = CONSTANT_BUILTIN_FUNCTIONS.contains(functionName.toString().toUpperCase());
19913                if (result) {
19914                        return true;
19915                }
19916                return false;
19917        }
19918
19919        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TParseTreeNode gspObject) {
19920                List<TExpression> directExpressions = new ArrayList<TExpression>();
19921                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
19922                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
19923                if (gspObject instanceof TFunctionCall) {
19924                        TFunctionCall functionCall = (TFunctionCall) gspObject;
19925                        getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
19926                } else if (gspObject instanceof TCaseExpression) {
19927                        TCaseExpression expr = (TCaseExpression) gspObject;
19928                        TExpression inputExpr = expr.getInput_expr();
19929                        if (inputExpr != null) {
19930                                if(option.isShowCaseWhenAsDirect()){
19931                                        directExpressions.add(inputExpr);
19932                                }
19933                                else {
19934                                        conditionExpressions.add(inputExpr);
19935                                }
19936                        }
19937                        TExpression defaultExpr = expr.getElse_expr();
19938                        if (defaultExpr != null) {
19939                                directExpressions.add(defaultExpr);
19940                        }
19941                        TWhenClauseItemList list = expr.getWhenClauseItemList();
19942                        for (int i = 0; i < list.size(); i++) {
19943                                TWhenClauseItem element = list.getWhenClauseItem(i);
19944                                if(option.isShowCaseWhenAsDirect()){
19945                                        directExpressions.add(element.getComparison_expr());
19946                                }
19947                                else {
19948                                        conditionExpressions.add(element.getComparison_expr());
19949                                }
19950                                directExpressions.add(element.getReturn_expr());
19951                        }
19952                }
19953
19954                for (int j = 0; j < directExpressions.size(); j++) {
19955                        columnsInExpr visitor = new columnsInExpr();
19956                        directExpressions.get(j).inOrderTraverse(visitor);
19957
19958                        List<TObjectName> objectNames = visitor.getObjectNames();
19959                        List<TParseTreeNode> functions = visitor.getFunctions();
19960
19961                        if (functions != null && !functions.isEmpty()) {
19962                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
19963                        }
19964
19965                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
19966                        if (subquerys != null && !subquerys.isEmpty()) {
19967                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
19968                        }
19969
19970                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
19971
19972                        List<TParseTreeNode> constants = visitor.getConstants();
19973                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
19974                }
19975
19976                conditionExpressions.addAll(indirectExpressions);
19977                for (int j = 0; j < conditionExpressions.size(); j++) {
19978                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
19979                }
19980        }
19981        
19982        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TCallStatement callStatment, int argumentIndex, Process process) {
19983                List<TExpression> directExpressions = new ArrayList<TExpression>();
19984                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
19985                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
19986
19987                getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex);
19988
19989                for (int j = 0; j < directExpressions.size(); j++) {
19990                        columnsInExpr visitor = new columnsInExpr();
19991                        directExpressions.get(j).inOrderTraverse(visitor);
19992
19993                        List<TObjectName> objectNames = visitor.getObjectNames();
19994                        List<TParseTreeNode> functions = visitor.getFunctions();
19995
19996                        if (functions != null && !functions.isEmpty()) {
19997                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
19998                        }
19999
20000                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20001                        if (subquerys != null && !subquerys.isEmpty()) {
20002                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20003                        }
20004
20005                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20006                        if (relation != null) {
20007                                relation.setProcess(process);
20008                        }
20009
20010                        List<TParseTreeNode> constants = visitor.getConstants();
20011                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20012                }
20013
20014                conditionExpressions.addAll(indirectExpressions);
20015                for (int j = 0; j < conditionExpressions.size(); j++) {
20016                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
20017                }
20018        }
20019
20020        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TFunctionCall functionCall, int argumentIndex, Process process) {
20021                List<TExpression> directExpressions = new ArrayList<TExpression>();
20022                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
20023                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
20024
20025                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentIndex);
20026
20027                for (int j = 0; j < directExpressions.size(); j++) {
20028                        columnsInExpr visitor = new columnsInExpr();
20029                        directExpressions.get(j).inOrderTraverse(visitor);
20030
20031                        List<TObjectName> objectNames = visitor.getObjectNames();
20032                        List<TParseTreeNode> functions = visitor.getFunctions();
20033
20034                        if (functions != null && !functions.isEmpty()) {
20035                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
20036                        }
20037
20038                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20039                        if (subquerys != null && !subquerys.isEmpty()) {
20040                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20041                        }
20042
20043                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20044                        if (relation != null) {
20045                                relation.setProcess(process);
20046                        }
20047
20048                        List<TParseTreeNode> constants = visitor.getConstants();
20049                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20050                }
20051
20052                conditionExpressions.addAll(indirectExpressions);
20053                for (int j = 0; j < conditionExpressions.size(); j++) {
20054                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
20055                }
20056        }
20057
20058        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TMssqlExecute functionCall, String argumentName, int argumentIndex, Process process) {
20059                List<TExpression> directExpressions = new ArrayList<TExpression>();
20060                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
20061                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
20062
20063                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentName, argumentIndex);
20064
20065                for (int j = 0; j < directExpressions.size(); j++) {
20066                        columnsInExpr visitor = new columnsInExpr();
20067                        directExpressions.get(j).inOrderTraverse(visitor);
20068
20069                        List<TObjectName> objectNames = visitor.getObjectNames();
20070                        List<TParseTreeNode> functions = visitor.getFunctions();
20071
20072                        if (functions != null && !functions.isEmpty()) {
20073                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
20074                        }
20075
20076                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20077                        if (subquerys != null && !subquerys.isEmpty()) {
20078                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20079                        }
20080
20081                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20082                        if (relation != null) {
20083                                relation.setProcess(process);
20084                        }
20085
20086                        List<TParseTreeNode> constants = visitor.getConstants();
20087                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20088                }
20089
20090                conditionExpressions.addAll(indirectExpressions);
20091                for (int j = 0; j < conditionExpressions.size(); j++) {
20092                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
20093                }
20094        }
20095
20096
20097        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20098                        TFunctionCall functionCall) {
20099                if (functionCall.getArgs() != null) {
20100                        for (int k = 0; k < functionCall.getArgs().size(); k++) {
20101                                TExpression expr = functionCall.getArgs().getExpression(k);
20102                                if(FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
20103                                        directExpressions.add(expr);
20104                                }
20105                                if(FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
20106                                        if("DECODE".equalsIgnoreCase(functionCall.getFunctionName().toString()) && option.getVendor() == EDbVendor.dbvoracle) {
20107                                                if(option.isShowCaseWhenAsDirect()) {
20108                                                        directExpressions.add(expr);
20109                                                        continue;
20110                                                }
20111                                        }
20112                                        indirectExpressions.add(expr);
20113                                }
20114                        }
20115                }
20116                if (functionCall.getXMLElementValueExprList() != null) {
20117                        for (int k = 0; k < functionCall.getXMLElementValueExprList().size(); k++) {
20118                                TExpression expr = functionCall.getXMLElementValueExprList().getResultColumn(k).getExpr();
20119                                directExpressions.add(expr);
20120                        }
20121                }
20122                if (functionCall.getTrimArgument() != null) {
20123                        TTrimArgument args = functionCall.getTrimArgument();
20124                        TExpression expr = args.getStringExpression();
20125                        if (expr != null) {
20126                                directExpressions.add(expr);
20127                        }
20128                        expr = args.getTrimCharacter();
20129                        if (expr != null) {
20130                                directExpressions.add(expr);
20131                        }
20132                }
20133
20134                if (functionCall.getAgainstExpr() != null) {
20135                        directExpressions.add(functionCall.getAgainstExpr());
20136                }
20137//              if (functionCall.getBetweenExpr() != null) {
20138//                      directExpressions.add(functionCall.getBetweenExpr());
20139//              }
20140                if (functionCall.getExpr1() != null) {
20141                        directExpressions.add(functionCall.getExpr1());
20142                }
20143                if (functionCall.getExpr2() != null) {
20144                        directExpressions.add(functionCall.getExpr2());
20145                }
20146                if (functionCall.getExpr3() != null) {
20147                        directExpressions.add(functionCall.getExpr3());
20148                }
20149                if (functionCall.getParameter() != null) {
20150                        directExpressions.add(functionCall.getParameter());
20151                }
20152                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getPartitionClause() != null) {
20153                        TExpressionList args = functionCall.getWindowDef().getPartitionClause().getExpressionList();
20154                        if (args != null) {
20155                                for (int k = 0; k < args.size(); k++) {
20156                                        TExpression expr = args.getExpression(k);
20157                                        if (expr != null) {
20158                                                indirectExpressions.add(expr);
20159                                        }
20160                                }
20161                        }
20162                }
20163                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getOrderBy() != null) {
20164                        TOrderByItemList orderByList = functionCall.getWindowDef().getOrderBy().getItems();
20165                        for (int i = 0; i < orderByList.size(); i++) {
20166                                TOrderByItem element = orderByList.getOrderByItem(i);
20167                                TExpression expression = element.getSortKey();
20168                                indirectExpressions.add(expression);
20169                        }
20170                }
20171                if (functionCall.getWithinGroup() != null && functionCall.getWithinGroup().getOrderBy() != null) {
20172                        TOrderByItemList orderByList = functionCall.getWithinGroup().getOrderBy().getItems();
20173                        for (int i = 0; i < orderByList.size(); i++) {
20174                                TOrderByItem element = orderByList.getOrderByItem(i);
20175                                TExpression expression = element.getSortKey();
20176                                indirectExpressions.add(expression);
20177                        }
20178                }
20179                if (functionCall.getCallTarget() != null) {
20180                        directExpressions.add(functionCall.getCallTarget().getExpr());
20181                }
20182                if (functionCall.getFieldValues() != null) {
20183                        for (int k = 0; k < functionCall.getFieldValues().size(); k++) {
20184                                TExpression expr = functionCall.getFieldValues().getResultColumn(k).getExpr();
20185                                directExpressions.add(expr);
20186                        }
20187                }
20188                if (functionCall instanceof TJsonObjectFunction) {
20189                        TJsonObjectFunction jsonObject = (TJsonObjectFunction)functionCall;
20190                        for (int k = 0; k < jsonObject.getKeyValues().size(); k++) {
20191                                TExpression expr = jsonObject.getKeyValues().get(k).getValue();
20192                                directExpressions.add(expr);
20193                        }
20194                }
20195                if (functionCall.getGroupConcatParam() != null) {
20196                        for (int k = 0; k < functionCall.getGroupConcatParam().getExprList().size(); k++) {
20197                                TExpression expr = functionCall.getGroupConcatParam().getExprList().getExpression(k);
20198                                directExpressions.add(expr);
20199                        }
20200                }
20201        }
20202
20203        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20204                                                                                TFunctionCall functionCall, int argumentIndex) {
20205                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
20206                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
20207                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
20208                                directExpressions.add(expr);
20209                        }
20210                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
20211                                indirectExpressions.add(expr);
20212                        }
20213                }
20214        }
20215        
20216        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20217                        TCallStatement functionCall, int argumentIndex) {
20218                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
20219                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
20220                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
20221                                        functionCall.getArgs().size(), argumentIndex)) {
20222                                directExpressions.add(expr);
20223                        }
20224                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
20225                                        functionCall.getArgs().size(), argumentIndex)) {
20226                                indirectExpressions.add(expr);
20227                        }
20228                }
20229        }
20230
20231        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20232                                                                                TMssqlExecute functionCall, String argumentName, int argumentIndex) {
20233                if (functionCall.getParameters() != null) {
20234                        for (int i = 0; i < functionCall.getParameters().size(); i++) {
20235                                TExecParameter param = functionCall.getParameters().getExecParameter(i);
20236                                if (param.getParameterName() != null) {
20237                                        if (DlineageUtil.compareColumnIdentifier(param.getParameterName().toString(), argumentName)) {
20238                                                TExpression expr = param.getParameterValue();
20239                                                directExpressions.add(expr);
20240                                        }
20241                                } else if (i == argumentIndex) {
20242                                        TExpression expr = param.getParameterValue();
20243                                        directExpressions.add(expr);
20244                                }
20245                        }
20246                }
20247        }
20248
20249        private void analyzeJoin(TJoin join, EffectType effectType) {
20250                if (join.getJoinItems() != null) {
20251                        for (int j = 0; j < join.getJoinItems().size(); j++) {
20252                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
20253                                TExpression expr = joinItem.getOnCondition();
20254                                if (expr != null) {
20255                                        analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on, effectType);
20256                                }
20257                        }
20258                }
20259
20260                if (join.getJoin() != null) {
20261                        analyzeJoin(join.getJoin(), effectType);
20262                }
20263        }
20264
20265        private TSelectSqlStatement getParentSetSelectStmt(TSelectSqlStatement stmt) {
20266                TCustomSqlStatement parent = stmt.getParentStmt();
20267                if (parent == null)
20268                        return null;
20269                if (parent.getStatements() != null) {
20270                        for (int i = 0; i < parent.getStatements().size(); i++) {
20271                                TCustomSqlStatement temp = parent.getStatements().get(i);
20272                                if (temp instanceof TSelectSqlStatement) {
20273                                        TSelectSqlStatement select = (TSelectSqlStatement) temp;
20274                                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
20275                                                return select;
20276                                }
20277                        }
20278                }
20279                if (parent instanceof TSelectSqlStatement) {
20280                        TSelectSqlStatement select = (TSelectSqlStatement) parent;
20281                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
20282                                return select;
20283                }
20284                return null;
20285        }
20286
20287        private void createSelectSetResultColumns(SelectSetResultSet resultSet, TSelectSqlStatement stmt) {
20288                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
20289                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
20290                } else {
20291                        TResultColumnList columnList = stmt.getResultColumnList();
20292                        ResultSet subqueryResultSet = (ResultSet) modelManager.getModel(columnList);
20293                        if(subqueryResultSet!=null && subqueryResultSet.isDetermined()) {
20294                                for (int j = 0; j < subqueryResultSet.getColumns().size(); j++) {
20295                                        ResultColumn tableColumn = subqueryResultSet.getColumns().get(j);
20296                                        if (tableColumn.getRefColumnName() != null) {
20297                                                TObjectName columnName = new TObjectName();
20298                                                columnName.setString(tableColumn.getRefColumnName());
20299                                                modelFactory.createDeterminedResultColumn(
20300                                                                resultSet, columnName);
20301                                        } else {
20302                                                TObjectName columnName = new TObjectName();
20303                                                columnName.setString(tableColumn.getName());
20304                                                modelFactory.createDeterminedResultColumn(
20305                                                                resultSet, columnName);
20306                                        }
20307                                }
20308                                resultSet.setDetermined(true);
20309                                return;
20310                        }
20311                        
20312                        boolean isDetermined = true;
20313                        for (int i = 0; i < columnList.size(); i++) {
20314                                TResultColumn column = columnList.getResultColumn(i);
20315                                
20316                                if ("*".equals(column.getColumnNameOnly())) {
20317                                        TObjectName columnObject = column.getFieldAttr();
20318                                        TTable sourceTable = columnObject.getSourceTable();
20319                                        if (sourceTable != null) {
20320                                                Object tableModel = modelManager.getModel(sourceTable);
20321                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
20322                                                        Table table = (Table) tableModel;
20323                                                        for (int j = 0; j < table.getColumns().size(); j++) {
20324                                                                TableColumn tableColumn = table.getColumns().get(j);
20325                                                                if (column.getExceptColumnList() != null) {
20326                                                                        boolean except = false;
20327                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
20328                                                                                if (getColumnName(objectName.toString())
20329                                                                                                .equals(getColumnName(tableColumn.getName()))) {
20330                                                                                        except = true;
20331                                                                                        break;
20332                                                                                }
20333                                                                        }
20334                                                                        if (!except && tableColumn.isStruct()) {
20335                                                                                List<String> names = SQLUtil
20336                                                                                                .parseNames(tableColumn.getName());
20337                                                                                for (String name : names) {
20338                                                                                        for (TObjectName objectName : column
20339                                                                                                        .getExceptColumnList()) {
20340                                                                                                if (getColumnName(objectName.toString())
20341                                                                                                                .equals(getColumnName(name))) {
20342                                                                                                        except = true;
20343                                                                                                        break;
20344                                                                                                }
20345                                                                                        }
20346                                                                                        if (except) {
20347                                                                                                break;
20348                                                                                        }
20349                                                                                }
20350                                                                        }
20351                                                                        if (except) {
20352                                                                                continue;
20353                                                                        }
20354                                                                }
20355                                                                TObjectName columnName = new TObjectName();
20356                                                                columnName.setString(tableColumn.getName());
20357                                                                ResultColumn resultColumn = modelFactory.createResultColumn(
20358                                                                                resultSet, columnName);
20359                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20360                                                                relation.setEffectType(EffectType.select);
20361                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20362                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
20363                                                        }
20364                                                        continue;
20365                                                } else if (tableModel instanceof ResultSet
20366                                                                && ((ResultSet) tableModel).isDetermined()) {
20367                                                        ResultSet table = (ResultSet) tableModel;
20368                                                        for (int j = 0; j < table.getColumns().size(); j++) {
20369                                                                ResultColumn tableColumn = table.getColumns().get(j);
20370                                                                if (column.getExceptColumnList() != null) {
20371                                                                        boolean except = false;
20372                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
20373                                                                                if (getColumnName(objectName.toString())
20374                                                                                                .equals(getColumnName(tableColumn.getName()))) {
20375                                                                                        except = true;
20376                                                                                        break;
20377                                                                                }
20378                                                                        }
20379                                                                        if (!except && tableColumn.isStruct()) {
20380                                                                                List<String> names = SQLUtil
20381                                                                                                .parseNames(tableColumn.getName());
20382                                                                                for (String name : names) {
20383                                                                                        for (TObjectName objectName : column
20384                                                                                                        .getExceptColumnList()) {
20385                                                                                                if (getColumnName(objectName.toString())
20386                                                                                                                .equals(getColumnName(name))) {
20387                                                                                                        except = true;
20388                                                                                                        break;
20389                                                                                                }
20390                                                                                        }
20391                                                                                        if (except) {
20392                                                                                                break;
20393                                                                                        }
20394                                                                                }
20395                                                                        }
20396                                                                        if (except) {
20397                                                                                continue;
20398                                                                        }
20399                                                                }
20400                                                                if (tableColumn.getRefColumnName() != null) {
20401                                                                        TObjectName columnName = new TObjectName();
20402                                                                        columnName.setString(tableColumn.getRefColumnName());
20403                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
20404                                                                                        resultSet, columnName);
20405                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20406                                                                        relation.setEffectType(EffectType.select);
20407                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20408                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20409                                                                } else {
20410                                                                        TObjectName columnName = new TObjectName();
20411                                                                        columnName.setString(tableColumn.getName());
20412                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
20413                                                                                        resultSet, columnName);
20414                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20415                                                                        relation.setEffectType(EffectType.select);
20416                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20417                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20418                                                                }
20419                                                        }
20420                                                        continue;
20421                                                }
20422                                                else {
20423                                                        isDetermined = false;
20424                                                }
20425                                        }
20426                                }
20427                                        
20428                                ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
20429
20430                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
20431                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
20432                                        if (columnObject.getFieldAttr() != null) {
20433                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
20434                                                        TObjectName fieldAttr = columnObject.getFieldAttr();
20435                                                        TTable sourceTable = fieldAttr.getSourceTable();
20436                                                        if (fieldAttr.getTableToken() != null && sourceTable != null) {
20437                                                                TObjectName[] columns = modelManager.getTableColumns(sourceTable);
20438                                                                for (int j = 0; j < columns.length; j++) {
20439                                                                        TObjectName columnName = columns[j];
20440                                                                        if (columnName == null) {
20441                                                                                continue;
20442                                                                        }
20443                                                                        if ("*".equals(getColumnName(columnName))) {
20444                                                                                continue;
20445                                                                        }
20446                                                                        resultColumn.bindStarLinkColumn(columnName);
20447                                                                }
20448
20449                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
20450                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
20451                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
20452                                                                                for (int z = 0; z < tableModel.getColumns().size(); z++) {
20453                                                                                        if ("*".equals(
20454                                                                                                        getColumnName(tableModel.getColumns().get(z).getColumnObject()))) {
20455                                                                                                continue;
20456                                                                                        }
20457                                                                                        resultColumn.bindStarLinkColumn(
20458                                                                                                        tableModel.getColumns().get(z).getColumnObject());
20459                                                                                }
20460                                                                        }
20461                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
20462                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
20463                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
20464                                                                                for (ResultColumn item : tableModel.getColumns()) {
20465                                                                                        if (item.hasStarLinkColumn()) {
20466                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
20467                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
20468                                                                                                                continue;
20469                                                                                                        }
20470                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
20471                                                                                                }
20472                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
20473                                                                                                TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
20474                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
20475                                                                                                        continue;
20476                                                                                                }
20477                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
20478                                                                                        }
20479                                                                                }
20480                                                                        }
20481                                                                }
20482
20483                                                        } else {
20484                                                                TTableList tables = stmt.getTables();
20485                                                                for (int k = 0; k < tables.size(); k++) {
20486                                                                        TTable tableElement = tables.getTable(k);
20487                                                                        TObjectName[] columns = modelManager.getTableColumns(tableElement);
20488                                                                        for (int j = 0; j < columns.length; j++) {
20489                                                                                TObjectName columnName = columns[j];
20490                                                                                if (columnName == null) {
20491                                                                                        continue;
20492                                                                                }
20493                                                                                if ("*".equals(getColumnName(columnName))) {
20494                                                                                        if (modelManager.getModel(tableElement) instanceof Table) {
20495                                                                                                Table tableModel = (Table) modelManager.getModel(tableElement);
20496                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
20497                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
20498                                                                                                                resultColumn.bindStarLinkColumn(
20499                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
20500                                                                                                        }
20501                                                                                                }
20502                                                                                        } else if (modelManager.getModel(tableElement) instanceof QueryTable) {
20503                                                                                                QueryTable tableModel = (QueryTable) modelManager
20504                                                                                                                .getModel(tableElement);
20505                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
20506                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
20507                                                                                                                if (item.hasStarLinkColumn()) {
20508                                                                                                                        for (TObjectName starLinkColumn : item
20509                                                                                                                                        .getStarLinkColumnList()) {
20510                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
20511                                                                                                                        }
20512                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
20513                                                                                                                        resultColumn.bindStarLinkColumn(
20514                                                                                                                                        (TObjectName) item.getColumnObject());
20515                                                                                                                }
20516                                                                                                        }
20517                                                                                                }
20518                                                                                        }
20519                                                                                        continue;
20520                                                                                }
20521                                                                                resultColumn.bindStarLinkColumn(columnName);
20522                                                                        }
20523                                                                }
20524                                                        }
20525                                                }
20526                                        }
20527                                }
20528                                
20529                                resultSet.setDetermined(isDetermined);
20530                        }
20531                }
20532        }
20533
20534        private void analyzeResultColumn(TResultColumn column, EffectType effectType) {
20535                TExpression expression = column.getExpr();
20536                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
20537                        expression = expression.getRightOperand();
20538                }
20539
20540                if (expression.getExpressionType() == EExpressionType.array_t) {
20541                        if (expression.getExprList() != null) {
20542                                for (TExpression expr : expression.getExprList()) {
20543                                        columnsInExpr visitor = new columnsInExpr();
20544                                        expr.inOrderTraverse(visitor);
20545                                        List<TObjectName> objectNames = visitor.getObjectNames();
20546
20547                                        List<TParseTreeNode> functions = visitor.getFunctions();
20548
20549                                        if (functions != null && !functions.isEmpty()) {
20550                                                analyzeFunctionDataFlowRelation(column, functions, effectType);
20551                                        }
20552
20553                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20554                                        if (subquerys != null && !subquerys.isEmpty()) {
20555                                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
20556                                        }
20557
20558                                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
20559
20560                                        List<TParseTreeNode> constants = visitor.getConstants();
20561                                        Object columnObject = modelManager.getModel(column);
20562                                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
20563
20564                                        analyzeRecordSetRelation(functions, effectType);
20565                                        // analyzeResultColumnImpact( column, effectType, functions);
20566                                }
20567                        }
20568                        else {
20569                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
20570                                TConstant constant = new TConstant();
20571                                constant.setString(expression.toString());
20572                                constants.add(constant);
20573                                Object columnObject = modelManager.getModel(column);
20574                                analyzeConstantDataFlowRelation(columnObject, constants, effectType, null);
20575                        }
20576                } else {
20577                        columnsInExpr visitor = new columnsInExpr();
20578                        expression.inOrderTraverse(visitor);
20579                        List<TObjectName> objectNames = visitor.getObjectNames();
20580
20581                        List<TParseTreeNode> functions = visitor.getFunctions();
20582
20583                        if (functions != null && !functions.isEmpty()) {
20584                                analyzeFunctionDataFlowRelation(column, functions, effectType);
20585                        }
20586
20587                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20588                        if (subquerys != null && !subquerys.isEmpty()) {
20589                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
20590                        }
20591
20592                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
20593
20594                        List<TParseTreeNode> constants = visitor.getConstants();
20595                        Object columnObject = modelManager.getModel(column);
20596                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
20597
20598                        analyzeRecordSetRelation(functions, effectType);
20599                        // analyzeResultColumnImpact( column, effectType, functions);
20600                }
20601        }
20602        
20603        
20604        private void analyzeValueColumn(Object object, TResultColumn column, EffectType effectType) {
20605                TExpression expression = column.getExpr();
20606                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
20607                        expression = expression.getRightOperand();
20608                }
20609
20610                if (expression.getExpressionType() == EExpressionType.array_t) {
20611                        if (expression.getExprList() != null) {
20612                                for (TExpression expr : expression.getExprList()) {
20613                                        columnsInExpr visitor = new columnsInExpr();
20614                                        expr.inOrderTraverse(visitor);
20615                                        List<TObjectName> objectNames = visitor.getObjectNames();
20616                                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
20617                                        List<TParseTreeNode> constants = visitor.getConstants();
20618                                        analyzeConstantDataFlowRelation(object, constants, effectType, null);
20619                                }
20620                        }
20621                        else {
20622                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
20623                                TConstant constant = new TConstant();
20624                                constant.setString(expression.toString());
20625                                constants.add(constant);
20626                                Object columnObject = modelManager.getModel(column);
20627                                analyzeConstantDataFlowRelation(object, constants, effectType, null);
20628                        }
20629                } else {
20630                        columnsInExpr visitor = new columnsInExpr();
20631                        expression.inOrderTraverse(visitor);
20632                        List<TObjectName> objectNames = visitor.getObjectNames();
20633                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
20634                        List<TParseTreeNode> constants = visitor.getConstants();
20635                        analyzeConstantDataFlowRelation(object, constants, effectType, null);           }
20636        }
20637
20638        private void analyzeTableColumn(TableColumn tableColumn, TFunctionCall functionCall, EffectType effectType) {
20639                List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
20640                functions.add(functionCall);
20641
20642                if (functions != null && !functions.isEmpty()) {
20643                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType);
20644                }
20645
20646                analyzeRecordSetRelation(functions, effectType);
20647        }
20648
20649        private void analyzeRecordSetRelation(List<TParseTreeNode> functions, EffectType effectType) {
20650                if (functions == null || functions.size() == 0)
20651                        return;
20652
20653                List<TFunctionCall> aggregateFunctions = new ArrayList<TFunctionCall>();
20654                for (TParseTreeNode function : functions) {
20655                        if (function instanceof TFunctionCall && isAggregateFunction((TFunctionCall) function)) {
20656                                aggregateFunctions.add((TFunctionCall) function);
20657                        }
20658                }
20659
20660                if (aggregateFunctions.size() == 0)
20661                        return;
20662
20663                for (int i = 0; i < aggregateFunctions.size(); i++) {
20664                        TFunctionCall function = aggregateFunctions.get(i);
20665
20666                        TCustomSqlStatement stmt = stmtStack.peek();
20667                        if (stmt instanceof TSelectSqlStatement) {
20668                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
20669                                if (select.getGroupByClause() != null) {
20670                                        TGroupByItemList groupByList = select.getGroupByClause().getItems();
20671                                        for (int j = 0; j < groupByList.size(); j++) {
20672                                                TGroupByItem groupBy = groupByList.getGroupByItem(j);
20673                                                TExpression expr = groupBy.getExpr();
20674                                                analyzeAggregate(function, expr);
20675                                        }
20676
20677                                        if (select.getGroupByClause().getHavingClause() != null) {
20678                                                analyzeAggregate(function, select.getGroupByClause().getHavingClause());
20679                                        }
20680                                        // if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString()))
20681                                        {
20682                                                analyzeAggregate(function, null);
20683                                        }
20684                                } else {
20685                                        analyzeAggregate(function, null);
20686                                }
20687                        }
20688                }
20689        }
20690
20691        private void analyzeDataFlowRelation(TParseTreeNode gspObject, List<TObjectName> objectNames,
20692                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions) {
20693                Object columnObject = modelManager.getModel(gspObject);
20694                analyzeDataFlowRelation(columnObject, objectNames, exceptColumnList, effectType, functions, null);
20695        }
20696
20697        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
20698                        List<TParseTreeNode> functions) {
20699                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, null);
20700        }
20701        
20702        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
20703                        List<TParseTreeNode> functions, Process process) {
20704                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, process);
20705        }
20706
20707        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
20708                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process) {
20709                return analyzeDataFlowRelation(modelObject, objectNames, exceptColumnList, effectType, functions, process, null);
20710        }
20711        
20712        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
20713                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process, Integer valueIndex) {
20714                if (objectNames == null || objectNames.size() == 0)
20715                        return null;
20716
20717                boolean isStar = false;
20718                boolean showStar = false;
20719
20720                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20721                relation.setEffectType(effectType);
20722                relation.setProcess(process);
20723
20724                if (functions != null && !functions.isEmpty()) {
20725                        relation.setFunction(getFunctionName(functions.get(0)));
20726                }
20727
20728                int columnIndex = -1;
20729
20730                boolean isOut = false;
20731
20732                if (modelObject instanceof ResultColumn) {
20733                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
20734
20735                        if ("*".equals(((ResultColumn) modelObject).getName())) {
20736                                isStar = true;
20737                                showStar = ((ResultColumn) modelObject).isShowStar();
20738                        }
20739
20740                        if (((ResultColumn) modelObject).getResultSet() != null) {
20741                                columnIndex = ((ResultColumn) modelObject).getResultSet().getColumns().indexOf(modelObject);
20742                        }
20743                } else if (modelObject instanceof TableColumn) {
20744                        Table table  = ((TableColumn) modelObject).getTable();
20745                        if(table.getSubType() == SubType.out && isNotInProcedure(table)){
20746                                isOut = true;
20747                                relation.addSource(new TableColumnRelationshipElement((TableColumn) modelObject));
20748                        }
20749                        else {
20750                                relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
20751                        }
20752
20753                        if ("*".equals(((TableColumn) modelObject).getName())) {
20754                                isStar = true;
20755                        }
20756
20757                        if (((TableColumn) modelObject).getTable() != null) {
20758                                columnIndex = ((TableColumn) modelObject).getTable().getColumns().indexOf(modelObject);
20759                        }
20760                } else {
20761                        throw new UnsupportedOperationException();
20762                }
20763
20764                for (int i = 0; i < objectNames.size(); i++) {
20765                        TObjectName columnName = objectNames.get(i);
20766                        if (columnName.toString().indexOf(".") == -1 && isConstant(columnName)) {
20767                                boolean isConstant = true;
20768                                if (columnName.getSourceTable() != null) {
20769                                        Table tableModel = modelManager.getTableByName(
20770                                                        DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
20771                                        if (tableModel != null && tableModel.getColumns() != null) {
20772                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
20773                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
20774                                                                        getColumnName(tableModel.getColumns().get(j).getName()))) {
20775                                                                isConstant = false;
20776                                                                break;
20777                                                        }
20778                                                }
20779                                        }
20780                                }
20781
20782                                if (isConstant) {
20783                                        if (option.isShowConstantTable()) {
20784                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
20785                                                TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
20786                                                if(tableColumn!=null) {
20787                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
20788                                                }
20789                                        }
20790                                        continue;
20791                                }
20792                        }
20793                        if (columnName.getDbObjectType() == EDbObjectType.variable) {
20794                                boolean find = false;
20795                                List<String> segments = SQLUtil.parseNames(columnName.toString());
20796                                if (segments.size() == 1) {
20797                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
20798                                        if (variable != null) {
20799                                                TableColumn columnModel = variable.getColumns().get(0);
20800                                                if(variable.getColumns().size()>0){
20801                                                        TableColumn matchColumn = matchColumn(variable.getColumns(), columnName);
20802                                                        if(matchColumn!=null){
20803                                                                columnModel = matchColumn;
20804                                                        }
20805                                                }
20806                                                if(isOut){
20807                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
20808                                                }
20809                                                else {
20810                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
20811                                                }
20812                                                find = true;
20813                                        } else {
20814                                                if (columnName.toString().matches("\\$\\d+")) {
20815                                                        TCustomSqlStatement stmt = stmtStack.peek();
20816                                                        Procedure procedure = modelManager
20817                                                                        .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
20818                                                        if(procedure!=null) {
20819                                                                Variable cursorVariable = modelFactory.createVariable(procedure.getArguments().get(Integer.valueOf(columnName.toString().replace("$", "")) - 1).getName());
20820                                                                if (isOut) {
20821                                                                        relation.setTarget(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
20822                                                                } else {
20823                                                                        relation.addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
20824                                                                }
20825                                                        }
20826                                                } else {
20827                                                        Variable cursorVariable = modelFactory.createVariable(columnName);
20828                                                        cursorVariable.setCreateTable(true);
20829                                                        cursorVariable.setSubType(SubType.record);
20830                                                        TableColumn variableProperty = null;
20831                                                        if(cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
20832                                variableProperty = modelFactory.createTableColumn(cursorVariable, columnName,
20833                                        true);
20834                            }
20835                                                        else{
20836                                                                variableProperty = cursorVariable.getColumns().get(0);
20837                                                        }
20838                                                        if (isOut) {
20839                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
20840                                                        } else {
20841                                                                relation.addSource(new TableColumnRelationshipElement(variableProperty));
20842                                                        }
20843                                                }
20844                                                find = true;
20845                                        }
20846                                } else if (option.getVendor() == EDbVendor.dbvoracle && columnName.getTableToken()!=null) {
20847                                        Variable cursorVariable = modelFactory
20848                                                        .createVariable(columnName.getTableToken().toString());
20849
20850                                        TObjectName variableColumnName = new TObjectName();
20851                                        variableColumnName.setString(segments.get(segments.size() - 1));
20852
20853                                        if (cursorVariable.getColumns() != null) {
20854                                                for (int j = 0; j < cursorVariable.getColumns().size(); j++) {
20855                                                        if (getColumnName(variableColumnName)
20856                                                                        .equals(getColumnName(cursorVariable.getColumns().get(j).getColumnObject()))) {
20857                                                                TableColumn columnModel = cursorVariable.getColumns().get(j);
20858                                                                if (isOut) {
20859                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
20860                                                                } else {
20861                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
20862                                                                }
20863                                                                find = true;
20864                                                        }
20865                                                }
20866                                        }
20867
20868                                        if (!find) {
20869                                                TableColumn variableColumn = new TableColumn(cursorVariable, variableColumnName);
20870                                                cursorVariable.addColumn(variableColumn);
20871                                                if (isOut) {
20872                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
20873                                                } else {
20874                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
20875                                                }
20876                                        }
20877                                } else {
20878                                        Table variable = modelManager
20879                                                        .getTableByName(DlineageUtil.getTableFullName(segments.get(segments.size() - 2)));
20880                                        if (variable != null) {
20881                                                for (int j = 0; j < variable.getColumns().size(); j++) {
20882                                                        if (getColumnName(columnName)
20883                                                                        .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
20884                                                                TableColumn columnModel = variable.getColumns().get(j);
20885                                                                if (isOut) {
20886                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
20887                                                                } else {
20888                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
20889                                                                }
20890                                                                find = true;
20891                                                        }
20892                                                }
20893                                        }
20894                                }
20895                                if (!find) {
20896                                        TCustomSqlStatement stmt = stmtStack.peek();
20897                                        if (getProcedureParentName(stmt) != null) {
20898                                                Procedure procedure = modelManager
20899                                                                .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
20900                                                if (procedure != null && procedure.getArguments() != null) {
20901                                                        for (Argument argument : procedure.getArguments()) {
20902                                                                if (DlineageUtil.getTableFullName(argument.getName())
20903                                                                                .equals(DlineageUtil.getTableFullName(columnName.toString()))) {
20904                                                                        relation.addSource(new ArgumentRelationshipElement(argument));
20905                                                                }
20906                                                        }
20907                                                }
20908                                        }
20909                                }
20910                                continue;
20911                        }
20912                        
20913                        // Handle sequence pseudocolumn syntax (sequence.NEXTVAL or sequence.CURRVAL)
20914                        // Used by Oracle, Snowflake, and accepted by other vendors for compatibility
20915                        if(("NEXTVAL".equalsIgnoreCase(columnName.getColumnNameOnly()) || "CURRVAL".equalsIgnoreCase(columnName.getColumnNameOnly()))){
20916                                List<String> segments = SQLUtil.parseNames(columnName.toString());
20917                                if (segments.size() > 1) {
20918                                        segments.remove(segments.size()-1);
20919                                        Table table = modelFactory.createTableByName(SQLUtil.mergeSegments(segments, 0), true);
20920                                        table.setSequence(true);
20921                                        TableColumn seqCursor = modelFactory.createTableColumn(table, columnName, true);
20922                                        relation.addSource(new TableColumnRelationshipElement(seqCursor));
20923                                        continue;
20924                                }
20925                        }
20926
20927                        {
20928                                if (columnName.getSourceTable() != null) {
20929
20930                                }
20931                                else {
20932                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
20933                                        if (variable == null) {
20934                                                variable = modelManager
20935                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableString()));
20936                                                if (variable != null && variable.isCursor()) {
20937                                                        TableColumn variableColumn = modelFactory.createInsertTableColumn(variable, columnName);
20938                                                        if (variableColumn != null) {
20939                                                                if(isOut){
20940                                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
20941                                                                }
20942                                                                else {
20943                                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
20944                                                                }
20945                                                        } else {
20946                                                                if(isOut){
20947                                                                        relation.setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0)));
20948                                                                }
20949                                                                else {
20950                                                                        relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0)));
20951                                                                }
20952                                                        }
20953                                                        continue;
20954                                                }
20955                                        } else if (variable.isVariable() || variable.isCursor()) {
20956                                                TableColumn columnModel = variable.getColumns().get(0);
20957                                                if (valueIndex != null) {
20958                                                        if(isOut){
20959                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel, valueIndex));
20960                                                        }
20961                                                        else {
20962                                                                relation.addSource(new TableColumnRelationshipElement(columnModel, valueIndex));
20963                                                        }
20964                                                } else {
20965                                                        if(isOut){
20966                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
20967                                                        }
20968                                                        else {
20969                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
20970                                                        }
20971                                                }
20972                                                continue;
20973                                        }
20974                                }
20975                        }
20976                        
20977                        if (columnName.getColumnNameOnly().startsWith("@")
20978                                        && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
20979                                continue;
20980                        }
20981
20982                        if (columnName.getColumnNameOnly().startsWith(":")
20983                                        && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
20984                                Table variable = modelManager
20985                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getColumnNameOnly().replace(":", "")));
20986                                if (variable != null) {
20987                                        for (int j = 0; j < variable.getColumns().size(); j++) {
20988                                                if (getColumnName(columnName).replace(":", "")
20989                                                                .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
20990                                                        TableColumn columnModel = variable.getColumns().get(j);
20991                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
20992                                                }
20993                                        }
20994                                }
20995                                continue;
20996                        }
20997
20998                        boolean linkedFirstTable = false;
20999                        
21000                        TCustomSqlStatement stmt = stmtStack.peek();
21001                        TTableList tableList = stmt.tables;
21002                        if ((tableList == null || tableList.size() == 0) && (hiveFromTables != null && hiveFromTables.size() > 0)) {
21003                                tableList = hiveFromTables;
21004                        }
21005
21006                        List<TTable> tables = new ArrayList<TTable>();
21007                        {
21008                                TTable table = columnName.getSourceTable();
21009
21010                                // 针对CursorVariable特殊处理
21011                                if (columnName.getTableToken() != null) {
21012
21013                                        Table tableModel = null;
21014                                        if (table != null && modelManager.getModel(table) instanceof Table) {
21015                                                tableModel = (Table) modelManager.getModel(table);
21016                                        }
21017
21018                                        if (tableModel == null) {
21019                                                tableModel = modelManager
21020                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableToken().toString()));
21021                                        }
21022
21023                                        if (tableModel == null) {
21024                                                TCustomSqlStatement currentStmt = ModelBindingManager.getGlobalStmtStack().peek();
21025                                                String procedureName = DlineageUtil.getProcedureParentName(currentStmt);
21026                                                String variableString = columnName.getTableToken().toString();
21027                                                if (variableString.startsWith(":")) {
21028                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
21029                                                }
21030                                                if (!SQLUtil.isEmpty(procedureName)) {
21031                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
21032                                                }
21033
21034                                                if (modelManager
21035                                                                .getTableByName(DlineageUtil.getTableFullName(variableString)) instanceof Variable) {
21036                                                        tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
21037                                                }
21038                                        }
21039
21040                                        if (tableModel != null) {
21041
21042                                                if (table == null) {
21043                                                        table = tableModel.getTableObject();
21044                                                }
21045
21046                                                if (tableModel.isVariable()) {
21047                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
21048                                                                TObjectName[] columns = modelManager.getTableColumns(table);
21049                                                                for (int j = 0; j < columns.length; j++) {
21050                                                                        TObjectName objectName = columns[j];
21051                                                                        if (objectName == null || "*".equals(getColumnName(objectName))) {
21052                                                                                continue;
21053                                                                        }
21054                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
21055                                                                                        false);
21056                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21057                                                                }
21058                                                        } else {
21059                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
21060
21061                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
21062                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
21063                                                                                if (exceptColumnList != null) {
21064                                                                                        boolean flag = false;
21065                                                                                        for (TObjectName objectName : exceptColumnList) {
21066                                                                                                if (getColumnName(objectName)
21067                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
21068                                                                                                        flag = true;
21069                                                                                                        break;
21070                                                                                                }
21071                                                                                        }
21072                                                                                        if (flag) {
21073                                                                                                continue;
21074                                                                                        }
21075                                                                                }
21076                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
21077                                                                        }
21078
21079                                                                        if (isStar && showStar) {
21080                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
21081                                                                                                false);
21082                                                                                if (columnModel == null) {
21083                                                                                        if (tableModel.isCreateTable()) {
21084                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
21085                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21086                                                                                                        relation.setShowStarRelation(true);
21087                                                                                                }
21088                                                                                        }
21089                                                                                } else {
21090                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21091                                                                                        relation.setShowStarRelation(true);
21092                                                                                }
21093                                                                        }
21094                                                                } else {
21095                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
21096                                                                                        false);
21097                                                                        
21098                                                                        if(columnModel == null && containStarColumn(tableModel.getColumns())){
21099                                                                                columnModel = getStarColumn(tableModel.getColumns());
21100                                                                                if (columnModel != null && tableModel.getSubType() == SubType.record_type) {
21101                                                                                        if(!"*".equals(getColumnName(columnName))){
21102                                                                                                columnModel.bindStarLinkColumn(columnName);
21103                                                                                        }
21104                                                                                }
21105                                                                        }
21106                                                                        
21107                                                                        if (columnModel == null) {
21108                                                                                continue;
21109                                                                        }
21110                                                                        if (columnModel.hasStarLinkColumn()) {
21111                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel,
21112                                                                                                columnModel.getStarLinkColumnNames()
21113                                                                                                                .indexOf(DlineageUtil.getColumnName(columnName))));
21114                                                                        } else {
21115                                                                                if (isOut) {
21116                                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
21117                                                                                } else {
21118                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21119                                                                                }
21120                                                                        }
21121                                                                        if (columnName.getSourceTable() != null
21122                                                                                        && columnName.getSourceTable().getTableType() == ETableSource.function) {
21123                                                                                analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
21124                                                                                                effectType);
21125                                                                        }
21126                                                                }
21127                                                        }
21128                                                        continue;
21129                                                }
21130                                        }
21131                                }
21132
21133                                if (table == null) {
21134                                        table = modelManager.getTable(stmt, columnName);
21135                                }
21136
21137                                if (table == null) {
21138                                        if (columnName.getTableToken() != null || !"*".equals(getColumnName(columnName))) {
21139                                                table = columnName.getSourceTable();
21140                                        }
21141                                        
21142                                        if (table == null && !SQLUtil.isEmpty(columnName.getTableString())) {
21143                                                table = modelManager.getTableFromColumn(columnName);
21144                                        }
21145                                }
21146
21147                                if (table == null) {
21148                                        if (tableList != null) {
21149                                                for (int j = 0; j < tableList.size(); j++) {
21150                                                        if (table != null)
21151                                                                break;
21152
21153                                                        TTable tTable = tableList.getTable(j);
21154                                                        if (tTable.getTableType().name().startsWith("open")) {
21155                                                                continue;
21156                                                        }
21157
21158                                                        if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
21159                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
21160                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
21161                                                                        if ("*".equals(getColumnName(refer)))
21162                                                                                continue;
21163                                                                        // For BigQuery struct field access, match base column name from FieldPath
21164                                                                        String structFullName = getStructFieldFullName(columnName);
21165                                                                        if (structFullName != null) {
21166                                                                                String baseName = getStructFieldBaseName(columnName);
21167                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(getColumnName(refer))
21168                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
21169                                                                                        table = tTable;
21170                                                                                        break;
21171                                                                                }
21172                                                                        }
21173                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
21174                                                                                table = tTable;
21175                                                                                break;
21176                                                                        }
21177                                                                }
21178                                                        }
21179
21180                                                        if (tTable.getLinkTable() != null) {
21181                                                                tTable = tTable.getLinkTable();
21182                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
21183                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
21184                                                                        if ("*".equals(getColumnName(refer)))
21185                                                                                continue;
21186                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
21187                                                                                table = tTable;
21188                                                                                break;
21189                                                                        }
21190                                                                }
21191                                                        }
21192
21193                                                        if (table != null)
21194                                                                break;
21195
21196                                                        if (columnName.getTableToken() != null && (columnName.getTableToken().getAstext()
21197                                                                        .equalsIgnoreCase(tTable.getName())
21198                                                                        || columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getAliasName()))) {
21199                                                                table = tTable;
21200                                                                break;
21201                                                        }
21202                                                }
21203                                        }
21204
21205                                        if (table == null) {
21206                                                for (int j = 0; j < tableList.size(); j++) {
21207                                                        if (table != null)
21208                                                                break;
21209
21210                                                        TTable tTable = tableList.getTable(j);
21211                                                        Object model = ModelBindingManager.get().getModel(tTable);
21212                                                        if (model instanceof Table) {
21213                                                                Table tableModel = (Table) model;
21214                                                                for (int z = 0; tableModel.getColumns() != null
21215                                                                                && z < tableModel.getColumns().size(); z++) {
21216                                                                        TableColumn refer = tableModel.getColumns().get(z);
21217                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
21218                                                                                table = tTable;
21219                                                                                break;
21220                                                                        }
21221                                                                        if (refer.hasStarLinkColumn()) {
21222                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
21223                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
21224                                                                                                table = tTable;
21225                                                                                                break;
21226                                                                                        }
21227                                                                                }
21228                                                                        }
21229                                                                }
21230                                                        } else if (model instanceof QueryTable) {
21231                                                                QueryTable tableModel = (QueryTable) model;
21232                                                                for (int z = 0; tableModel.getColumns() != null
21233                                                                                && z < tableModel.getColumns().size(); z++) {
21234                                                                        ResultColumn refer = tableModel.getColumns().get(z);
21235                                                                        // Try FieldPath-based matching first
21236                                                                        String structFullName = getStructFieldFullName(columnName);
21237                                                                        if (structFullName != null) {
21238                                                                                String baseName = getStructFieldBaseName(columnName);
21239                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(refer.getName())
21240                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
21241                                                                                        table = tTable;
21242                                                                                        break;
21243                                                                                }
21244                                                                        }
21245                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());
21246                                                                        if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
21247                                                                                if (DlineageUtil.getIdentifierNormalColumnName(refer.getName())
21248                                                                                                .equals(DlineageUtil
21249                                                                                                                .getIdentifierNormalColumnName(getColumnName(splits.get(0))))) {
21250                                                                                        table = tTable;
21251                                                                                        break;
21252                                                                                }
21253                                                                        }
21254                                                                        else if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
21255                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
21256                                                                                table = tTable;
21257                                                                                break;
21258                                                                        }
21259                                                                        if (refer.hasStarLinkColumn()) {
21260                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
21261                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
21262                                                                                                table = tTable;
21263                                                                                                break;
21264                                                                                        }
21265                                                                                }
21266                                                                        }
21267                                                                }
21268                                                        }
21269                                                }
21270                                        }
21271                                }
21272
21273                                if (columnName.getTableToken() == null && "*".equals(getColumnName(columnName))) {
21274                                        if (!hasJoin(stmt)) {
21275                                                tables.add(table);
21276                                        } else {
21277                                                for (int j = 0; j < tableList.size(); j++) {
21278                                                        tables.add(tableList.getTable(j));
21279                                                }
21280                                        }
21281                                } else if (table != null) {
21282                                        tables.add(table);
21283                                }
21284
21285                                // 此处特殊处理，多表关联无法找到 column 所属的 Table, tTable.getLinkedColumns
21286                                // 也找不到，退而求其次采用第一个表
21287
21288                                if (stmt.getParentStmt() != null && isApplyJoin(stmt.getParentStmt())
21289                                                && (tableList != null && tableList.size() > 0)) {
21290                                        stmt = stmt.getParentStmt();
21291                                        TTable applyTable = tableList.getTable(0);
21292                                        if (modelManager.getModel(table) == null) {
21293                                                modelFactory.createTable(applyTable);
21294                                        }
21295                                }
21296
21297                                if (columnName.toString().indexOf(".")==-1 && isConstant(columnName)) {
21298                                        boolean isConstant = true;
21299                                        if (columnName.getSourceTable() != null) {
21300                                                Table tableModel = modelManager.getTableByName(
21301                                                                DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
21302                                                if (tableModel != null && tableModel.getColumns() != null) {
21303                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
21304                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
21305                                                                                getColumnName(tableModel.getColumns().get(j).getName()))) {
21306                                                                        isConstant = false;
21307                                                                        break;
21308                                                                }
21309                                                        }
21310                                                }
21311                                        }
21312
21313                                        if (isConstant) {
21314                                                if (option.isShowConstantTable()) {
21315                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
21316                                                        TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
21317                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21318                                                }
21319                                                continue;
21320                                        }
21321                                }
21322
21323                                if (tableList != null && tableList.size() != 0 && tables.size() == 0
21324                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
21325                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
21326                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
21327                                                boolean find = false;
21328                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
21329                                                        if(resultColumn.equals(modelObject)) {
21330                                                                continue;
21331                                                        }
21332                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
21333                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
21334                                                                        if (resultColumn.getColumnObject() != null) {
21335                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
21336                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
21337                                                                                if (columnName.getStartToken().posinlist >= startToken
21338                                                                                                && columnName.getEndToken().posinlist <= endToken) {
21339                                                                                        continue;
21340                                                                                }
21341                                                                        }
21342                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
21343                                                                        find = true;
21344                                                                        break;
21345                                                                }
21346                                                        }
21347                                                }
21348                                                if (find) {
21349                                                        continue;
21350                                                }
21351                                        }
21352                                        
21353                                        TObjectName pseudoTableName = new TObjectName();
21354                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
21355                                        // Otherwise fall back to default pseudo table name
21356                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
21357                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
21358                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
21359                                        pseudoTable.setPseudo(true);
21360                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
21361
21362                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
21363                                        // add the pseudo table column as source
21364                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
21365                                                if (isOut) {
21366                                                        relation.setTarget(new TableColumnRelationshipElement(pseudoTableColumn));
21367                                                } else {
21368                                                        relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
21369                                                }
21370                                        }
21371
21372                                        if (isLinkOrphanColumnToFirstTable()) {
21373                                                TTable orphanTable = tableList.getTable(0);
21374                                                tables.add(orphanTable);
21375                                                Object tableModel = modelManager.getModel(orphanTable);
21376                                                if (tableModel == null) {
21377                                                        if(orphanTable.getSubquery()!=null) {
21378                                                                QueryTable queryTable = modelFactory.createQueryTable(orphanTable);
21379                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
21380                                                                analyzeSelectStmt(subquery);
21381                                                        }
21382                                                        else {
21383                                                                tableModel = modelFactory.createTable(orphanTable);
21384                                                        }
21385                                                }
21386                                                if (tableModel instanceof Table) {
21387                                                        TableColumn orphanColum = modelFactory.createTableColumn((Table) tableModel, columnName, false);
21388                                                        if(orphanColum!=null) {
21389                                                                ErrorInfo errorInfo = new ErrorInfo();
21390                                                                errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
21391                                                                errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
21392                                                                                + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
21393                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
21394                                                                                columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
21395                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
21396                                                                                columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
21397                                                                                ModelBindingManager.getGlobalHash()));
21398                                                                errorInfo.fillInfo(this);
21399                                                                errorInfos.add(errorInfo);
21400                                                        }
21401                                                        if (orphanTable.getSubquery() != null) {
21402                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
21403                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
21404                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
21405                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
21406                                                                        if(sourceTable instanceof Table) {
21407                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
21408                                                                        }
21409                                                                        else if(sourceTable instanceof ResultSet) {
21410                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
21411                                                                        }
21412                                                                }
21413                                                        }
21414                                                        else if (orphanTable.getCTE()!=null && orphanTable.getCTE().getSubquery() != null) {
21415                                                                TSelectSqlStatement subquery = orphanTable.getCTE().getSubquery();
21416                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
21417                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
21418                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
21419                                                                        if(sourceTable instanceof Table) {
21420                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
21421                                                                        }
21422                                                                        else if(sourceTable instanceof ResultSet) {
21423                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
21424                                                                        }
21425                                                                }
21426                                                        }
21427                                                }
21428                                                
21429                                                linkedFirstTable = true;
21430                                        }
21431                                }
21432                        }
21433
21434                        for (int k = 0; k < tables.size(); k++) {
21435                                TTable table = tables.get(k);
21436                                if (table != null) {
21437                                        Object object = modelManager.getModel(table);
21438                                        if(object instanceof PivotedTable) {
21439                                                TPivotClause clause = (TPivotClause)((PivotedTable)object).getGspObject();
21440                                                if(clause.getAliasClause()!=null) {
21441                                                        object = modelManager.getModel(clause.getAliasClause());
21442                                                }
21443                                        }
21444                                        if (object == null && table.getTableType() == ETableSource.objectname) {
21445                                                if (table.getCTE() != null) {
21446                                                        QueryTable queryTable = modelFactory.createQueryTable(table);
21447                                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
21448                                                        analyzeSelectStmt(subquery);
21449                                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
21450
21451                                                        if (resultSetModel != null && resultSetModel != queryTable
21452                                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21453                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21454                                                                impactRelation.setEffectType(EffectType.select);
21455                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21456                                                                                resultSetModel.getRelationRows()));
21457                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21458                                                                                queryTable.getRelationRows()));
21459                                                        }
21460
21461                                                        if (resultSetModel != null && resultSetModel != queryTable) {
21462                                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
21463                                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
21464                                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
21465                                                                                        sourceColumn);
21466
21467                                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
21468                                                                        queryRalation.setEffectType(EffectType.select);
21469                                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
21470                                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
21471                                                                }
21472                                                        }
21473                                                        
21474                                                        object = queryTable;
21475                                                        
21476                                                } else {
21477                                                        object = modelFactory.createTable(table);
21478                                                }
21479                                        }
21480                                        if (object instanceof Function) {
21481                                                relation.addSource(new ResultColumnRelationshipElement(((Function)object).getColumns().get(0)));
21482                                                continue;
21483                                        } else if (object instanceof ResultSet && !(object instanceof QueryTable)) {
21484                                                //Object tableModel = modelManager.getModel(columnName.getSourceTable());
21485                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
21486                                                                (ResultSet)object);
21487                                                if(table.getTableType() == ETableSource.function && !relation.getSources().isEmpty()) {
21488                                                        relation.getSources().stream().reduce((first, second) -> second).get().addTransform(Transform.FUNCTION, table);
21489                                                }
21490                                                continue;
21491                                        }
21492                                        else if (object instanceof Table) {
21493                                                Table tableModel = (Table) modelManager.getModel(table);
21494                                                if (tableModel != null) {
21495                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
21496                                                                TObjectName[] columns = modelManager.getTableColumns(table);
21497                                                                for (int j = 0; j < columns.length; j++) {
21498                                                                        TObjectName objectName = columns[j];
21499                                                                        if (objectName == null || ("*".equals(getColumnName(objectName)) && tableModel.isDetermined())) {
21500                                                                                continue;
21501                                                                        }
21502                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
21503                                                                                        false);
21504                                                                        if(columnModel == null) {
21505                                                                                continue;
21506                                                                        }
21507                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21508                                                                }
21509                                                        } else {
21510                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
21511                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
21512                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
21513                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
21514                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
21515                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
21516                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
21517                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
21518                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
21519                                                                                        }
21520                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
21521                                                                                        if(tableModel.isDetermined()) {
21522                                                                                                resultSet.getColumns().clear();
21523                                                                                        }
21524                                                                                }
21525                                                                        }
21526                                                                        
21527                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
21528                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
21529                                                                                if (exceptColumnList != null) {
21530                                                                                        boolean flag = false;
21531                                                                                        for (TObjectName objectName : exceptColumnList) {
21532                                                                                                if (getColumnName(objectName)
21533                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
21534                                                                                                        flag = true;
21535                                                                                                        break;
21536                                                                                                }
21537                                                                                        }
21538                                                                                        if (flag) {
21539                                                                                                continue;
21540                                                                                        }
21541                                                                                }
21542                                                                                
21543                                                                                if (replaceAsIdentifierMap.containsKey(tableModel.getColumns().get(j).getName())) {
21544                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableModel.getColumns().get(j).getName());
21545                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
21546                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(tableModel.getColumns().get(j).getName()));
21547                                                                                        Transform transform = new Transform();
21548                                                                                        transform.setType(Transform.EXPRESSION);
21549                                                                                        TObjectName expression = new TObjectName();
21550                                                                                        expression.setString(expr.first);
21551                                                                                transform.setCode(expression);
21552                                                                                        resultColumn.setTransform(transform);
21553                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
21554                                                                                } else {
21555                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
21556                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
21557                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
21558                                                                                                                columnModel.getColumnObject());
21559                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
21560                                                                                                relation1.setEffectType(effectType);
21561                                                                                                relation1.setProcess(process);
21562                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
21563                                                                                                relation1.addSource(new TableColumnRelationshipElement(columnModel));
21564                                                                                        }
21565                                                                                        else {
21566                                                                                                relation.addSource(
21567                                                                                                        new TableColumnRelationshipElement(columnModel));
21568                                                                                        }
21569                                                                                }
21570                                                                        }
21571
21572                                                                        if (isStar && showStar) {
21573                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
21574                                                                                                false);
21575                                                                                if (columnModel == null) {
21576                                                                                        if(tableModel.isCreateTable()) {
21577                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
21578                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21579                                                                                                        relation.setShowStarRelation(true);
21580                                                                                                }
21581                                                                                        }
21582                                                                                } else {
21583                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21584                                                                                        relation.setShowStarRelation(true);
21585                                                                                }
21586                                                                        }
21587                                                                } else {
21588                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
21589                                                                                        false);
21590                                                                        if(columnModel == null) {
21591                                                                                if(tableModel.isCreateTable()) {
21592                                                                                        boolean flag = false;
21593                                                                                        // Try FieldPath-based matching first for BigQuery/Redshift struct fields
21594                                                                                        String structFullName = getStructFieldFullName(columnName);
21595                                                                                        if (structFullName != null && !flag) {
21596                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
21597                                                                                                        if (tableColumn.isStruct()
21598                                                                                                                && DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
21599                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(structFullName))) {
21600                                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
21601                                                                                                                flag = true;
21602                                                                                                                if (modelObject instanceof ResultColumn) {
21603                                                                                                                        ((ResultColumn) modelObject).setStruct(true);
21604                                                                                                                } else if (modelObject instanceof TableColumn) {
21605                                                                                                                        ((TableColumn) modelObject).setStruct(true);
21606                                                                                                                }
21607                                                                                                                break;
21608                                                                                                        }
21609                                                                                                }
21610                                                                                        }
21611                                                                                        if (ModelBindingManager.getGlobalVendor() == EDbVendor.dbvbigquery || ModelBindingManager.getGlobalVendor() == EDbVendor.dbvredshift) {
21612                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
21613                                                                                                        if(tableColumn.isStruct()) {
21614                                                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
21615                                                                                                                if (modelObject instanceof TableColumn) {
21616                                                                                                                        TableColumn targetColumn = (TableColumn) modelObject;
21617                                                                                                                        if (targetColumn.isStruct()) {
21618                                                                                                                                List<String> targetNames = SQLUtil
21619                                                                                                                                                .parseNames(targetColumn.getName());
21620                                                                                                                                if (!getColumnName(targetNames.get(0))
21621                                                                                                                                                .equals(getColumnName(names.get(0)))) {
21622                                                                                                                                        continue;
21623                                                                                                                                }
21624                                                                                                                        }
21625                                                                                                                }
21626                                                                                                                else if (modelObject instanceof ResultColumn) {
21627                                                                                                                        ResultColumn targetColumn = (ResultColumn) modelObject;
21628                                                                                                                        if (targetColumn.isStruct()) {
21629                                                                                                                                List<String> targetNames = SQLUtil
21630                                                                                                                                                .parseNames(targetColumn.getName());
21631                                                                                                                                if (!getColumnName(targetNames.get(0))
21632                                                                                                                                                .equals(getColumnName(names.get(0)))) {
21633                                                                                                                                        continue;
21634                                                                                                                                }
21635                                                                                                                        }
21636                                                                                                                }
21637                                                                                                                for(String name: names) {
21638                                                                                                                        if (getColumnName(name)
21639                                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
21640                                                                                                                                relation.addSource(
21641                                                                                                                                                new TableColumnRelationshipElement(tableColumn));
21642                                                                                                                                flag = true;
21643                                                                                                                                if (modelObject instanceof ResultColumn) {
21644                                                                                                                                        ((ResultColumn)modelObject).setStruct(true);
21645                                                                                                                                }
21646                                                                                                                                else if (modelObject instanceof TableColumn) {
21647                                                                                                                                        ((TableColumn)modelObject).setStruct(true);
21648                                                                                                                                }
21649                                                                                                                                break;
21650                                                                                                                        }
21651                                                                                                                }
21652
21653                                                                                                                if (!flag && tableModel.getColumns().size() == 1 && tableModel
21654                                                                                                                                .getColumns().get(0).getSourceColumn() != null) {
21655                                                                                                                        TableColumn sourceColumn = tableModel
21656                                                                                                                                        .getColumns().get(0).getSourceColumn();
21657                                                                                                                        Table sourceTable = sourceColumn.getTable();
21658                                                                                                                        TObjectName sourceColumnName = new TObjectName();
21659                                                                                                                        sourceColumnName.setString(sourceColumn.getName() + "."
21660                                                                                                                                        + columnName.getColumnNameOnly());
21661                                                                                                                        TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceColumnName, true);
21662                                                                                                                        relation.addSource(
21663                                                                                                                                        new TableColumnRelationshipElement(sourceTableColumn));
21664                                                                                                                        flag = true;
21665                                                                                                                        break;
21666                                                                                                                }
21667                                                                                                        }
21668                                                                                                        else if (getColumnName(tableColumn.getName())
21669                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
21670                                                                                                                relation.addSource(
21671                                                                                                                                new TableColumnRelationshipElement(tableColumn));
21672                                                                                                                flag = true;
21673                                                                                                                break;
21674                                                                                                        }
21675                                                                                                }
21676                                                                                                if (modelObject instanceof TableColumn) {
21677                                                                                                        TableColumn column = (TableColumn) modelObject;
21678                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
21679                                                                                                                if (tableColumn.getColumnIndex() == null) {
21680                                                                                                                        continue;
21681                                                                                                                }
21682                                                                                                                if (tableColumn.getName().toLowerCase()
21683                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
21684                                                                                                                                && tableColumn.getName().toLowerCase()
21685                                                                                                                                                .indexOf(column.getColumnObject().toString()
21686                                                                                                                                                                .toLowerCase()) == -1) {
21687                                                                                                                        continue;
21688                                                                                                                }
21689                                                                                                                flag = true;
21690                                                                                                                relation.addSource(
21691                                                                                                                                new TableColumnRelationshipElement(tableColumn));
21692                                                                                                                relation.setShowStarRelation(true);
21693                                                                                                        }
21694                                                                                                        if (flag)
21695                                                                                                                break;
21696
21697                                                                                                } else if (modelObject instanceof ResultColumn) {
21698                                                                                                        ResultColumn column = (ResultColumn) modelObject;
21699                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
21700                                                                                                                if (tableColumn.getColumnIndex() == null) {
21701                                                                                                                        continue;
21702                                                                                                                }
21703                                                                                                                if (tableColumn.getName().toLowerCase()
21704                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
21705                                                                                                                                && tableColumn.getName().toLowerCase()
21706                                                                                                                                                .indexOf(column.getColumnObject().toString()
21707                                                                                                                                                                .toLowerCase()) == -1) {
21708                                                                                                                        continue;
21709                                                                                                                }
21710                                                                                                                flag = true;
21711                                                                                                                relation.addSource(
21712                                                                                                                                new TableColumnRelationshipElement(tableColumn));
21713                                                                                                                relation.setShowStarRelation(true);
21714                                                                                                        }
21715                                                                                                        if (flag)
21716                                                                                                                break;
21717
21718                                                                                                }
21719                                                                                        }
21720                                                                                        if (!flag 
21721                                                                                                        && (isStar 
21722                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getName()))
21723                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getAlias())))) {
21724                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
21725                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21726                                                                                                        relation.setShowStarRelation(true);
21727                                                                                                }
21728                                                                                        }
21729                                                                                }
21730                                                                        }
21731                                                                        else {
21732                                                                                if (linkedFirstTable || columnModel.getCandidateParents() != null) {
21733                                                                                        if (columnName.getCandidateTables() != null
21734                                                                                                        && columnName.getCandidateTables().size() > 1) {
21735                                                                                                List<Object> candidateParents = new ArrayList<Object>();
21736                                                                                                for(TTable tableItem: columnName.getCandidateTables()) {
21737                                                                                                        Object model = modelManager.getModel(tableItem);
21738                                                                                                        if(model!=null) {
21739                                                                                                                candidateParents.add(model);
21740                                                                                                        }
21741                                                                                                }
21742                                                                                                if (candidateParents.size() > 1) {
21743                                                                                                        columnModel.setCandidateParents(candidateParents);
21744                                                                                                }
21745                                                                                        }
21746                                                                                }
21747                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
21748                                                                                relation.setShowStarRelation(true);
21749                                                                                if(modelObject instanceof TableColumn) {
21750                                                                                        TableColumn targetTableColumn = (TableColumn)modelObject;
21751                                                                                        if(targetTableColumn.getTable().getSubType() == SubType.unnest && targetTableColumn.getTable().getColumns().size() == 1) {
21752                                                                                                targetTableColumn.setSourceColumn(columnModel);
21753                                                                                                targetTableColumn.setStruct(true);
21754                                                                                        }
21755                                                                                }
21756                                                                                if (columnName.getSourceTable() != null
21757                                                                                                && columnName.getSourceTable().getTableType() == ETableSource.function) {
21758                                                                                        analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
21759                                                                                                        effectType);
21760                                                                                }
21761                                                                        }
21762                                                                }
21763                                                        }
21764                                                }
21765                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
21766                                                QueryTable queryTable = (QueryTable) modelManager.getModel(table);
21767
21768                                                TObjectNameList cteColumns = null;
21769                                                TSelectSqlStatement subquery = null;
21770                                                if (queryTable.getTableObject().getCTE() != null) {
21771                                                        subquery = queryTable.getTableObject().getCTE().getSubquery();
21772                                                        cteColumns = queryTable.getTableObject().getCTE().getColumnList();
21773                                                } else if (queryTable.getTableObject().getAliasClause() != null
21774                                                                && queryTable.getTableObject().getAliasClause().getColumns() != null) {
21775
21776                                                } else if (queryTable.getTableObject().getTableExpr() != null
21777                                                                && queryTable.getTableObject().getTableExpr().getSubQuery() != null) {
21778                                                        subquery = queryTable.getTableObject().getTableExpr().getSubQuery();
21779                                                } else {
21780                                                        subquery = queryTable.getTableObject().getSubquery();
21781                                                }
21782
21783                                                if (cteColumns != null) {
21784                                                        for (int j = 0; j < cteColumns.size(); j++) {
21785                                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
21786                                                        }
21787                                                }
21788
21789                                                if (subquery != null && subquery.isCombinedQuery()) {
21790                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
21791                                                                        .getModel(subquery);
21792
21793                                                        if (selectSetResultSetModel != null
21794                                                                        && !selectSetResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21795                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21796                                                                impactRelation.setEffectType(EffectType.select);
21797                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21798                                                                                selectSetResultSetModel.getRelationRows()));
21799                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21800                                                                                queryTable.getRelationRows()));
21801                                                        }
21802
21803                                                        if (selectSetResultSetModel != null) {
21804                                                                if (getColumnName(columnName).equals("*")) {
21805                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
21806                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
21807                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
21808                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
21809                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
21810                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
21811                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
21812                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
21813                                                                                        }
21814                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
21815                                                                                        if(selectSetResultSetModel.isDetermined()) {
21816                                                                                                resultSet.getColumns().clear();
21817                                                                                        }
21818                                                                                }
21819                                                                        }
21820                                                                        
21821                                                                        
21822                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
21823                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
21824                                                                                if (cteColumns != null) {
21825                                                                                        if (j < cteColumns.size()) {
21826                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
21827
21828                                                                                                if (exceptColumnList != null) {
21829                                                                                                        boolean flag = false;
21830                                                                                                        for (TObjectName objectName : exceptColumnList) {
21831                                                                                                                if (getColumnName(objectName)
21832                                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
21833                                                                                                                        flag = true;
21834                                                                                                                        break;
21835                                                                                                                }
21836                                                                                                        }
21837                                                                                                        if (flag) {
21838                                                                                                                continue;
21839                                                                                                        }
21840                                                                                                }
21841
21842                                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
21843                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
21844                                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
21845                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
21846                                                                                                        Transform transform = new Transform();
21847                                                                                                        transform.setType(Transform.EXPRESSION);
21848                                                                                                        TObjectName expression = new TObjectName();
21849                                                                                                        expression.setString(expr.first);
21850                                                                                                transform.setCode(expression);
21851                                                                                                        resultColumn.setTransform(transform);
21852                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
21853                                                                                                } else {
21854                                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
21855                                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
21856                                                                                                                TObjectName resultColumnName = new TObjectName();
21857                                                                                                                resultColumnName.setString(targetColumn.getName());
21858                                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
21859                                                                                                                                resultColumnName);
21860                                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
21861                                                                                                                relation1.setEffectType(effectType);
21862                                                                                                                relation1.setProcess(process);
21863                                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
21864                                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
21865                                                                                                        }
21866                                                                                                        else {
21867                                                                                                                relation.addSource(
21868                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
21869                                                                                                        }
21870                                                                                                }
21871                                                                                                
21872                                                                                        }
21873                                                                                } else {
21874                                                                                        ResultColumn targetColumn = modelFactory
21875                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
21876
21877                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
21878                                                                                                        .createDataFlowRelation();
21879                                                                                        combinedQueryRelation.setEffectType(effectType);
21880                                                                                        combinedQueryRelation
21881                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
21882                                                                                        combinedQueryRelation
21883                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
21884
21885                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
21886                                                                                }
21887                                                                        }
21888                                                                        break;
21889                                                                } else {
21890                                                                        boolean flag = false;
21891
21892                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
21893                                                                                ResultColumn sourceColumn = selectSetResultSetModel
21894                                                                                                .getColumns().get(j);
21895                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
21896                                                                                if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
21897                                                                                        if (getColumnName(sourceColumn.getName())
21898                                                                                                        .equalsIgnoreCase(getColumnName(splits.get(0)))
21899                                                                                                        || getColumnName(sourceColumn.getName())
21900                                                                                                                        .equalsIgnoreCase(getColumnName(splits.get(1)))) {
21901                                                                                                ResultColumn targetColumn = modelFactory
21902                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
21903
21904                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
21905                                                                                                                .createDataFlowRelation();
21906                                                                                                combinedQueryRelation.setEffectType(effectType);
21907                                                                                                combinedQueryRelation.setTarget(
21908                                                                                                                new ResultColumnRelationshipElement(targetColumn));
21909                                                                                                combinedQueryRelation.addSource(
21910                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
21911
21912                                                                                                relation.addSource(
21913                                                                                                                new ResultColumnRelationshipElement(targetColumn));
21914                                                                                                flag = true;
21915                                                                                                break;
21916                                                                                        }
21917                                                                                }
21918                                                                                else if (getColumnName(sourceColumn.getName())
21919                                                                                                .equalsIgnoreCase(getColumnName(columnName))) {
21920                                                                                        ResultColumn targetColumn = modelFactory
21921                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
21922
21923                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
21924                                                                                                        .createDataFlowRelation();
21925                                                                                        combinedQueryRelation.setEffectType(effectType);
21926                                                                                        combinedQueryRelation
21927                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
21928                                                                                        combinedQueryRelation
21929                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
21930
21931                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
21932                                                                                        flag = true;
21933                                                                                        break;
21934                                                                                }
21935                                                                                else if (sourceColumn instanceof SelectSetResultColumn && ((SelectSetResultColumn)sourceColumn).getAliasSet().size() > 1) {
21936                                                                                        for (String alias : ((SelectSetResultColumn)sourceColumn).getAliasSet()) {
21937                                                                                                if (getColumnName(alias).equalsIgnoreCase(getColumnName(columnName))) {
21938                                                                                                        ResultColumn targetColumn = modelFactory
21939                                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
21940
21941                                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
21942                                                                                                                        .createDataFlowRelation();
21943                                                                                                        combinedQueryRelation.setEffectType(effectType);
21944                                                                                                        combinedQueryRelation.setTarget(
21945                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
21946                                                                                                        combinedQueryRelation.addSource(
21947                                                                                                                        new ResultColumnRelationshipElement(sourceColumn));
21948
21949                                                                                                        relation.addSource(
21950                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
21951                                                                                                        flag = true;
21952                                                                                                        break;
21953                                                                                                }
21954                                                                                        }
21955                                                                                        if (flag) {
21956                                                                                                break;
21957                                                                                        }
21958                                                                                }
21959                                                                        }
21960
21961                                                                        if (flag) {
21962                                                                                break;
21963                                                                        } else if (columnIndex != -1) {
21964                                                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
21965                                                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
21966                                                                                        if (!sourceColumn.getStarLinkColumns().isEmpty()) {
21967                                                                                                if (cteColumns != null) {
21968                                                                                                        if (j < cteColumns.size()) {
21969                                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
21970                                                                                                                relation.addSource(
21971                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
21972                                                                                                        }
21973                                                                                                } 
21974                                                                                                else {
21975                                                                                                        if(sourceColumn.hasStarLinkColumn()) {
21976                                                                                                                for (TObjectName linkColumn : sourceColumn.getStarLinkColumnList()) {
21977                                                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
21978                                                                                                                                ResultColumn targetColumn = modelFactory
21979                                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
21980
21981                                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
21982                                                                                                                                                .createDataFlowRelation();
21983                                                                                                                                combinedQueryRelation.setEffectType(effectType);
21984                                                                                                                                combinedQueryRelation.setTarget(
21985                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
21986                                                                                                                                combinedQueryRelation.addSource(
21987                                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
21988
21989                                                                                                                                relation.addSource(
21990                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
21991                                                                                                                                flag = true;
21992                                                                                                                                break;
21993                                                                                                                        }
21994                                                                                                                }
21995                                                                                                        }
21996                                                                                                        
21997                                                                                                        if(!flag) {
21998                                                                                                                ResultColumn targetColumn = modelFactory
21999                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
22000
22001                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
22002                                                                                                                                .createDataFlowRelation();
22003                                                                                                                combinedQueryRelation.setEffectType(effectType);
22004                                                                                                                combinedQueryRelation.setTarget(
22005                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22006                                                                                                                combinedQueryRelation.addSource(
22007                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
22008
22009                                                                                                                relation.addSource(
22010                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22011                                                                                                        }
22012                                                                                                }
22013                                                                                                flag = true;
22014                                                                                                break;
22015                                                                                        }
22016                                                                                }
22017                                                                        }
22018
22019                                                                        if (flag) {
22020                                                                                break;
22021                                                                        } else if (columnIndex < selectSetResultSetModel.getColumns().size()
22022                                                                                        && columnIndex != -1) {
22023                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns()
22024                                                                                                .get(columnIndex);
22025                                                                                if (cteColumns != null) {
22026                                                                                        boolean flag1 = false;
22027                                                                                        for (ResultColumn targetColumn : queryTable.getColumns()) {
22028                                                                                                if (getColumnName(targetColumn.getName())
22029                                                                                                                .equalsIgnoreCase(getColumnName(columnName))) {
22030                                                                                                        relation.addSource(
22031                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
22032                                                                                                        flag1 = true;
22033                                                                                                        break;
22034                                                                                                }
22035                                                                                        }
22036                                                                                        if (!flag1 && columnIndex < cteColumns.size()){
22037                                                                                                ResultColumn targetColumn = queryTable.getColumns()
22038                                                                                                                .get(columnIndex);
22039                                                                                                relation.addSource(
22040                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22041                                                                                        }
22042                                                                                } else {
22043                                                                                        ResultColumn targetColumn = modelFactory
22044                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
22045
22046                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
22047                                                                                                        .createDataFlowRelation();
22048                                                                                        combinedQueryRelation.setEffectType(effectType);
22049                                                                                        combinedQueryRelation
22050                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
22051                                                                                        combinedQueryRelation
22052                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
22053
22054                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
22055                                                                                }
22056                                                                                flag = true;
22057                                                                                break;
22058                                                                        }
22059
22060                                                                        if (flag) {
22061                                                                                break;
22062                                                                        }
22063                                                                }
22064                                                        } else if (cteColumns != null) {
22065                                                                if (getColumnName(columnName).equals("*")) {
22066                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22067                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22068                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
22069                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
22070                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
22071                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
22072                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22073                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22074                                                                                        }
22075                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22076                                                                                        
22077                                                                                        if (columnName.getSourceColumn() != null) {
22078                                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
22079                                                                                                if (model instanceof ResultColumn && ((ResultColumn)model).getResultSet().isDetermined()) {
22080                                                                                                        resultSet.getColumns().clear();
22081                                                                                                }
22082                                                                                        } else if (columnName.getSourceTable() != null) {
22083                                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
22084                                                                                                if (tableModel instanceof Table && ((Table)tableModel).isDetermined()) {
22085                                                                                                        resultSet.getColumns().clear();
22086                                                                                                }
22087                                                                                        }
22088                                                                                }
22089                                                                        }
22090                                                                        
22091                                                                        for (int j = 0; j < cteColumns.size(); j++) {
22092                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
22093
22094                                                                                if (exceptColumnList != null) {
22095                                                                                        boolean flag = false;
22096                                                                                        for (TObjectName objectName : exceptColumnList) {
22097                                                                                                if (getColumnName(objectName)
22098                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
22099                                                                                                        flag = true;
22100                                                                                                        break;
22101                                                                                                }
22102                                                                                        }
22103                                                                                        if (flag) {
22104                                                                                                continue;
22105                                                                                        }
22106                                                                                }
22107
22108                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
22109                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
22110                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22111                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
22112                                                                                        Transform transform = new Transform();
22113                                                                                        transform.setType(Transform.EXPRESSION);
22114                                                                                        TObjectName expression = new TObjectName();
22115                                                                                        expression.setString(expr.first);
22116                                                                                transform.setCode(expression);
22117                                                                                        resultColumn.setTransform(transform);
22118                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22119                                                                                } else {
22120                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
22121                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
22122                                                                                                TObjectName resultColumnName = new TObjectName();
22123                                                                                                resultColumnName.setString(targetColumn.getName());
22124                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
22125                                                                                                                resultColumnName);
22126                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
22127                                                                                                relation1.setEffectType(effectType);
22128                                                                                                relation1.setProcess(process);
22129                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
22130                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
22131                                                                                        }
22132                                                                                        else {
22133                                                                                                relation.addSource(
22134                                                                                                        new ResultColumnRelationshipElement(targetColumn));
22135                                                                                        }
22136                                                                                }
22137                                                                        }
22138                                                                        break;
22139                                                                } else {
22140                                                                        boolean flag = false;
22141
22142                                                                        for (int j = 0; j < cteColumns.size(); j++) {
22143                                                                                TObjectName sourceColumn = cteColumns.getObjectName(j);
22144
22145                                                                                if (getColumnName(sourceColumn).equalsIgnoreCase(getColumnName(columnName))) {
22146                                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
22147
22148                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
22149                                                                                        flag = true;
22150                                                                                        break;
22151                                                                                }
22152                                                                        }
22153
22154                                                                        if (flag) {
22155                                                                                break;
22156                                                                        }
22157                                                                }
22158                                                        }
22159
22160                                                        if (columnName.getSourceColumn() != null) {
22161                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
22162                                                                if (model instanceof ResultColumn) {
22163                                                                        ResultColumn resultColumn = (ResultColumn) model;
22164                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
22165                                                                }
22166                                                        } else if (columnName.getSourceTable() != null) {
22167                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
22168                                                                if (tableModel instanceof Table) {
22169                                                                        Object model = modelManager
22170                                                                                        .getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
22171                                                                        if (model instanceof TableColumn) {
22172                                                                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
22173                                                                        }
22174                                                                }
22175                                                        }
22176                                                } else {
22177                                                        List<ResultColumn> columns = queryTable.getColumns();
22178                                                        if (getColumnName(columnName).equals("*")) {
22179                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22180                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22181                                                                if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
22182                                                                        TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
22183                                                                        if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
22184                                                                                for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
22185                                                                                        replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22186                                                                                        replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22187                                                                                }
22188                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22189                                                                                if(queryTable.isDetermined()) {
22190                                                                                        resultSet.getColumns().clear();
22191                                                                                }
22192                                                                        }
22193                                                                }
22194                                                                
22195                                                                int index = 0;
22196                                                                for (int j = 0; j < queryTable.getColumns().size(); j++) {
22197                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
22198                                                                        if (exceptColumnList != null) {
22199                                                                                boolean flag = false;
22200                                                                                for (TObjectName objectName : exceptColumnList) {
22201                                                                                        if (getColumnName(objectName)
22202                                                                                                        .equals(getColumnName(targetColumn.getName()))) {
22203                                                                                                flag = true;
22204                                                                                                break;
22205                                                                                        }
22206                                                                                }
22207                                                                                if (flag) {
22208                                                                                        continue;
22209                                                                                }
22210                                                                        }
22211                                                                        
22212                                                                        if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
22213                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
22214                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22215                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
22216                                                                                Transform transform = new Transform();
22217                                                                                transform.setType(Transform.EXPRESSION);
22218                                                                                TObjectName expression = new TObjectName();
22219                                                                                expression.setString(expr.first);
22220                                                                        transform.setCode(expression);
22221                                                                                resultColumn.setTransform(transform);
22222                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22223                                                                        } else {
22224                                                                                if(!replaceAsIdentifierMap.isEmpty()) {
22225                                                                                        ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
22226                                                                                        TObjectName resultColumnName = new TObjectName();
22227                                                                                        resultColumnName.setString(targetColumn.getName());
22228                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
22229                                                                                                        resultColumnName);
22230                                                                                        DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
22231                                                                                        relation1.setEffectType(effectType);
22232                                                                                        relation1.setProcess(process);
22233                                                                                        relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
22234                                                                                        relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
22235                                                                                }
22236                                                                                else {
22237                                                                                        relation.addSource(
22238                                                                                                new ResultColumnRelationshipElement(targetColumn));
22239                                                                                }
22240                                                                        }
22241                                                                        index++;
22242                                                                }
22243                                                        } else {
22244                                                                if (table.getCTE() != null) {
22245                                                                        
22246                                                                        if (modelObject instanceof TableColumn) {
22247                                                                                Table modelTable = ((TableColumn) modelObject).getTable();
22248                                                                                if (modelTable.getSubType() == SubType.unnest) {
22249                                                                                        boolean find = false;
22250                                                                                        for (k = 0; k < columns.size(); k++) {
22251                                                                                                ResultColumn column = columns.get(k);
22252                                                                                                if (column.isStruct()) {
22253                                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
22254                                                                                                        for (String name : names) {
22255                                                                                                                if (getColumnName(name).equals(getColumnName(columnName))) {
22256                                                                                                                        DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
22257                                                                                                                        unnestRelation.setEffectType(effectType);
22258                                                                                                                        unnestRelation.setProcess(process);
22259                                                                                                                        unnestRelation.addSource(new ResultColumnRelationshipElement(
22260                                                                                                                                        column, columnName));
22261                                                                                                                        TObjectName unnestTableColumnName = new TObjectName();
22262                                                                                                                        unnestTableColumnName.setString(names.get(names.size()-1));
22263                                                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
22264                                                                                                                        unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
22265                                                                                                                        find = true;
22266                                                                                                                }
22267                                                                                                        }
22268                                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
22269                                                                                                        if (names.size() == 1 && names1.size() >= 1) {
22270                                                                                                                for (String name : names1) {
22271                                                                                                                        if (getColumnName(name)
22272                                                                                                                                        .equals(getColumnName(column.getName()))) {
22273                                                                                                                                DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
22274                                                                                                                                unnestRelation.setEffectType(effectType);
22275                                                                                                                                unnestRelation.setProcess(process);
22276                                                                                                                                unnestRelation.addSource(new ResultColumnRelationshipElement(
22277                                                                                                                                                column, columnName));
22278                                                                                                                                TObjectName unnestTableColumnName = new TObjectName();
22279                                                                                                                                unnestTableColumnName.setString(names1.get(names.size()-1));
22280                                                                                                                                TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
22281                                                                                                                                unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
22282                                                                                                                                find = true;
22283                                                                                                                        }
22284                                                                                                                }
22285                                                                                                        }
22286                                                                                                }
22287                                                                                        }
22288                                                                                        if (find) {
22289                                                                                                modelTable.getColumns().remove(modelObject);
22290                                                                                                break;
22291                                                                                        }
22292                                                                                }
22293                                                                        }
22294                                                                        
22295                                                                        for (k = 0; k < columns.size(); k++) {
22296                                                                                ResultColumn column = columns.get(k);
22297                                                                                if ("*".equals(column.getName())) {
22298                                                                                        if (!containsStarColumn(column, columnName)) {
22299                                                                                                column.bindStarLinkColumn(columnName);
22300                                                                                        }
22301                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
22302                                                                                } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
22303                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
22304                                                                                        if (!column.equals(modelObject)) {
22305                                                                                                relation.addSource(
22306                                                                                                                new ResultColumnRelationshipElement(column, columnName));
22307                                                                                        }
22308                                                                                        break;
22309                                                                                } else if(column.isStruct()) {
22310                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
22311                                                                                        for(String name: names) {
22312                                                                                                if (getColumnName(name)
22313                                                                                                                .equals(getColumnName(columnName))) {
22314                                                                                                        relation.addSource(
22315                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
22316                                                                                                }
22317                                                                                        }
22318                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
22319                                                                                        if (names.size() == 1 && names1.size() >= 1) {
22320                                                                                                for(String name: names1) {
22321                                                                                                        if (getColumnName(name)
22322                                                                                                                        .equals(getColumnName(column.getName()))) {
22323                                                                                                                relation.addSource(
22324                                                                                                                                new ResultColumnRelationshipElement(column, columnName));
22325                                                                                                        }
22326                                                                                                }
22327                                                                                        }
22328                                                                                }
22329                                                                        }
22330                                                                } else if (table.getAliasClause() != null
22331                                                                                && table.getAliasClause().getColumns() != null) {
22332                                                                        for (k = 0; k < columns.size(); k++) {
22333                                                                                ResultColumn column = columns.get(k);
22334                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
22335                                                                                if ("*".equals(column.getName())) {
22336                                                                                        if (!containsStarColumn(column, columnName)) {
22337                                                                                                column.bindStarLinkColumn(columnName);
22338                                                                                        }
22339                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
22340                                                                                } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
22341                                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
22342                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
22343                                                                                                if (!column.equals(modelObject)) {
22344                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column,
22345                                                                                                                        columnName));
22346                                                                                                }
22347                                                                                                break;
22348                                                                                        }
22349                                                                                } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
22350                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
22351                                                                                        if (!column.equals(modelObject)) {
22352                                                                                                relation.addSource(
22353                                                                                                                new ResultColumnRelationshipElement(column, columnName));
22354                                                                                        }
22355                                                                                        break;
22356                                                                                }
22357                                                                        }
22358                                                                } else if (table.getSubquery() != null || (table.getTableExpr() != null
22359                                                                                && table.getTableExpr().getSubQuery() != null)) {
22360                                                                        TSelectSqlStatement select = table.getSubquery();
22361                                                                        if (select == null) {
22362                                                                                select = table.getTableExpr().getSubQuery();
22363                                                                        }
22364                                                                        if (columnName.getSourceTable() != null) {
22365                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
22366                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
22367                                                                                                tableModel);
22368                                                                        } else if (columnName.getObjectToken() != null
22369                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
22370                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
22371                                                                                                table.getAliasName())) {
22372                                                                                        Object tableModel = modelManager.getModel(table);
22373                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
22374                                                                                                        columnName, tableModel);
22375                                                                                }
22376                                                                        } else if(columns!=null) {
22377                                                                                for (k = 0; k < columns.size(); k++) {
22378                                                                                        ResultColumn column = columns.get(k);
22379                                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());        
22380                                                                                        if ("*".equals(column.getName())) {
22381                                                                                                if (!containsStarColumn(column, columnName)) {
22382                                                                                                        column.bindStarLinkColumn(columnName);
22383                                                                                                }
22384                                                                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
22385                                                                                        } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
22386                                                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
22387                                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
22388                                                                                                        if (!column.equals(modelObject)) {
22389                                                                                                                relation.addSource(new ResultColumnRelationshipElement(column,
22390                                                                                                                                columnName));
22391                                                                                                        }
22392                                                                                                        break;
22393                                                                                                }
22394                                                                                        } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
22395                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
22396                                                                                                if (!column.equals(modelObject)) {
22397                                                                                                        relation.addSource(
22398                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
22399                                                                                                }
22400                                                                                                break;
22401                                                                                        }
22402                                                                                }
22403                                                                        }
22404                                                                } else if (table.getOutputMerge() != null) {
22405                                                                        if (columnName.getSourceColumn() != null) {
22406                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
22407                                                                                if (model instanceof ResultColumn) {
22408                                                                                        ResultColumn resultColumn = (ResultColumn) model;
22409                                                                                        if ("*".equals(resultColumn.getName())
22410                                                                                                        && !containsStarColumn(resultColumn, columnName)) {
22411                                                                                                resultColumn.bindStarLinkColumn(columnName);
22412                                                                                        }
22413                                                                                        relation.addSource(
22414                                                                                                        new ResultColumnRelationshipElement(resultColumn, columnName));
22415                                                                                }
22416                                                                        } else if (columnName.getSourceTable() != null) {
22417                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
22418                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
22419                                                                                                tableModel);
22420                                                                        } else if (columnName.getObjectToken() != null
22421                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
22422                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
22423                                                                                                table.getAliasName())) {
22424                                                                                        Object tableModel = modelManager.getModel(table);
22425                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
22426                                                                                                        columnName, tableModel);
22427                                                                                }
22428                                                                        }
22429                                                                }
22430                                                        }
22431                                                }
22432                                        }
22433                                }
22434                        }
22435                        if (relation.getSources().size() == 0 && isKeyword(columnName)) {
22436                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
22437                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable, columnName, true);
22438                                relation.addSource(new ConstantRelationshipElement(constantColumn));
22439                        }
22440
22441                        if (relation.getSources().size() > 0) {
22442                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
22443                                        Object source = sourceItem.getElement();
22444                                        ImpactRelationship impactRelation = null;
22445                                        if (source instanceof ResultColumn
22446                                                        && !((ResultColumn) source).getResultSet().getRelationRows().getHoldRelations().isEmpty()) {
22447                                                impactRelation = modelFactory.createImpactRelation();
22448                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
22449                                                                ((ResultColumn) source).getResultSet().getRelationRows()));
22450                                                impactRelation.setEffectType(effectType);
22451                                        } else if (source instanceof TableColumn
22452                                                        && !((TableColumn) source).getTable().getRelationRows().getHoldRelations().isEmpty()) {
22453                                                impactRelation = modelFactory.createImpactRelation();
22454                                                impactRelation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(
22455                                                                ((TableColumn) source).getTable().getRelationRows()));
22456                                                impactRelation.setEffectType(effectType);;
22457                                        }
22458
22459                                        if (impactRelation == null) {
22460                                                continue;
22461                                        }
22462
22463                                        Object target = relation.getTarget().getElement();
22464                                        if (target instanceof ResultColumn) {
22465                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
22466                                                                ((ResultColumn) target).getResultSet().getRelationRows()));
22467                                        } else if (target instanceof TableColumn) {
22468                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
22469                                                                ((TableColumn) target).getTable().getRelationRows()));
22470                                        }
22471
22472                                        if (impactRelation.getSources().iterator().next().getElement() == impactRelation.getTarget().getElement()) {
22473                                                modelManager.removeRelation(impactRelation);
22474                                        }
22475                                }
22476                        }
22477                }
22478                return relation;
22479        }
22480
22481        private boolean isNotInProcedure(Table table) {
22482        TStoredProcedureSqlStatement stmt = getProcedureParent(stmtStack.peek());
22483                Procedure procedure = this.modelFactory.createProcedure(stmt);
22484                if(procedure!=null && procedure.getName().equals(table.getParent())){
22485                        return false;
22486                }
22487                return true;
22488        }
22489
22490        private boolean hasJoin(TCustomSqlStatement stmt) {
22491                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
22492                        return false;
22493                if (stmt.getJoins().size() > 1) {
22494                        return true;
22495                }
22496                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
22497                if (joinItems == null || joinItems.size() == 0) {
22498                        return false;
22499                }
22500                return true;
22501        }
22502
22503        private boolean isInQuery(TSelectSqlStatement query, TResultColumn column) {
22504                if(query == null)
22505                        return false;
22506                TResultColumnList columns = query.getResultColumnList();
22507                if (columns != null) {
22508                        for (int i = 0; i < columns.size(); i++) {
22509                                if (columns.getResultColumn(i).equals(column)) {
22510                                        return true;
22511                                }
22512                        }
22513                }
22514                return false;
22515        }
22516
22517        private void appendResultColumnRelationSource(Object modelObject, DataFlowRelationship relation, int columnIndex,
22518                        TObjectName columnName, Object tableModel) {
22519                if (tableModel instanceof Table) {
22520                        Object model = modelManager.getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
22521                        if (model instanceof TableColumn) {
22522                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
22523                        }
22524                } else if (tableModel instanceof ResultSet) {
22525                        List<ResultColumn> queryColumns = ((ResultSet) tableModel).getColumns();
22526                        boolean flag = false;
22527                        for (int l = 0; l < queryColumns.size(); l++) {
22528                                ResultColumn column = queryColumns.get(l);
22529                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
22530                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
22531                                        if (!column.equals(modelObject)) {
22532                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
22533                                                flag = true;
22534                                        }
22535                                        break;
22536                                }
22537                        }
22538                        if (!flag) {
22539                                for (int l = 0; l < queryColumns.size(); l++) {
22540                                        ResultColumn column = queryColumns.get(l);
22541                                        if ("*".equals(column.getName())) {
22542                                                if (!containsStarColumn(column, columnName)) {
22543                                                        column.bindStarLinkColumn(columnName);
22544                                                }
22545                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
22546                                                flag = true;
22547                                                break;
22548                                        }
22549                                }
22550                        }
22551                        if (!flag && columnIndex < queryColumns.size() && columnIndex != -1) {
22552                                relation.addSource(new ResultColumnRelationshipElement(queryColumns.get(columnIndex), columnName));
22553                        }
22554                }
22555        }
22556
22557        private boolean isApplyJoin(TCustomSqlStatement stmt) {
22558                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
22559                        return false;
22560                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
22561                if (joinItems == null || joinItems.size() == 0) {
22562                        return false;
22563                }
22564                if (joinItems.getJoinItem(0).getJoinType() == EJoinType.crossapply
22565                                || joinItems.getJoinItem(0).getJoinType() == EJoinType.outerapply)
22566                        return true;
22567                return false;
22568        }
22569
22570        private boolean containsStarColumn(ResultColumn resultColumn, TObjectName columnName) {
22571                String targetColumnName = getColumnName(columnName);
22572                if (resultColumn.hasStarLinkColumn()) {
22573                        return resultColumn.getStarLinkColumns().containsKey(targetColumnName);
22574                }
22575                return false;
22576        }
22577
22578        private void analyzeAggregate(TFunctionCall function, TExpression expr) {
22579                TCustomSqlStatement stmt = stmtStack.peek();
22580                ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
22581                if (resultSet == null) {
22582                        return;
22583                }
22584
22585                if (expr != null) {
22586                        columnsInExpr visitor = new columnsInExpr();
22587                        expr.inOrderTraverse(visitor);
22588                        List<TObjectName> objectNames = visitor.getObjectNames();
22589                        for (int j = 0; j < objectNames.size(); j++) {
22590                                TObjectName columnName = objectNames.get(j);
22591
22592                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
22593                                        continue;
22594                                }
22595
22596                                if (columnName.getColumnNameOnly().startsWith("@")
22597                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
22598                                        continue;
22599                                }
22600
22601                                if (columnName.getColumnNameOnly().startsWith(":")
22602                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
22603                                        continue;
22604                                }
22605
22606                                if(modelManager.getModel(function.getFunctionName()) == null) {
22607                                        continue;
22608                                }
22609                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
22610                                relation.setEffectType(EffectType.function);
22611                                relation.setFunction(function.getFunctionName().toString());
22612                                relation.setTarget(new ResultColumnRelationshipElement(
22613                                                (ResultColumn) modelManager.getModel(function.getFunctionName())));
22614                                TTable table = modelManager.getTable(stmt, columnName);
22615                                if (table != null) {
22616                                        if (modelManager.getModel(table) instanceof Table) {
22617                                                Table tableModel = (Table) modelManager.getModel(table);
22618                                                if (tableModel != null) {
22619                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
22620                                                        if (columnModel != null) {
22621                                                                relation.addSource(
22622                                                                                new TableColumnRelationshipElement(columnModel, columnName.getLocation()));
22623                                                        }
22624                                                }
22625                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
22626                                                Object model = modelManager.getModel(columnName.getSourceColumn());
22627                                                if (model instanceof ResultColumn) {
22628                                                        ResultColumn resultColumn = (ResultColumn) model;
22629                                                        if (resultColumn != null) {
22630                                                                relation.addSource(
22631                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
22632                                                        }
22633                                                }
22634                                        }
22635                                }
22636                        }
22637
22638                        List<TParseTreeNode> functions = visitor.getFunctions();
22639                        for (int j = 0; j < functions.size(); j++) {
22640                                TParseTreeNode functionObj = functions.get(j);
22641                                Object functionModel = modelManager.getModel(functionObj);
22642                                if (functionModel == null) {
22643                                        functionModel = createFunction(functionObj);
22644                                }
22645                                if (functionModel instanceof Function) {
22646                                        AbstractRelationship relation;
22647                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
22648                                                // relation = modelFactory.createDataFlowRelation();
22649                                                relation = modelFactory.createRecordSetRelation();
22650                                        } else {
22651                                                relation = modelFactory.createRecordSetRelation();
22652                                        }
22653                                        relation.setEffectType(EffectType.function);
22654                                        relation.setFunction(function.getFunctionName().toString());
22655                                        relation.setTarget(new ResultColumnRelationshipElement(
22656                                                        (ResultColumn) modelManager.getModel(function.getFunctionName())));
22657
22658                                        if (functionObj instanceof TFunctionCall) {
22659                                                ResultColumn resultColumn = (ResultColumn) modelManager
22660                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
22661                                                if (resultColumn != null) {
22662                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
22663                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
22664                                                        relation.addSource(element);
22665                                                }
22666                                        }
22667                                        if (functionObj instanceof TCaseExpression) {
22668                                                ResultColumn resultColumn = (ResultColumn) modelManager
22669                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
22670                                                if (resultColumn != null) {
22671                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
22672                                                        relation.addSource(element);
22673                                                }
22674                                        }
22675                                } else if (functionModel instanceof Table) {
22676                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
22677                                                        function.getFunctionName(), false);
22678                                        AbstractRelationship relation;
22679                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
22680                                                // relation = modelFactory.createDataFlowRelation();
22681                                                relation = modelFactory.createRecordSetRelation();
22682                                        } else {
22683                                                relation = modelFactory.createRecordSetRelation();
22684                                        }
22685                                        relation.setEffectType(EffectType.function);
22686                                        relation.setFunction(function.getFunctionName().toString());
22687                                        relation.setTarget(new ResultColumnRelationshipElement(
22688                                                        (ResultColumn) modelManager.getModel(function.getFunctionName())));
22689                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
22690                                        relation.addSource(element);
22691                                }
22692                        }
22693                }
22694
22695                if (expr == null || "COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
22696                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())
22697                                        // https://e.gitee.com/gudusoft/issues/list?issue=I4L5EO
22698                                        // 对于非 count() 函数，当有group by clause时， RelationRows 不参与indirect dataflow,
22699                                        // 让位与group by clause中的column.
22700                                        || ((TSelectSqlStatement) stmt).getGroupByClause() == null) {
22701                                TTableList tables = stmt.getTables();
22702                                if (tables != null) {
22703                                        for (int i = 0; i < tables.size(); i++) {
22704                                                TTable table = tables.getTable(i);
22705                                                if (modelManager.getModel(table) == null && table.getSubquery() == null) {
22706                                                        modelFactory.createTable(table);
22707                                                }
22708                                                if (modelManager.getModel(table) instanceof Table) {
22709                                                        Table tableModel = (Table) modelManager.getModel(table);
22710                                                        AbstractRelationship relation;
22711                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
22712                                                                // relation = modelFactory.createDataFlowRelation();
22713                                                                relation = modelFactory.createRecordSetRelation();
22714                                                        } else {
22715                                                                relation = modelFactory.createRecordSetRelation();
22716                                                        }
22717                                                        relation.setEffectType(EffectType.function);
22718                                                        relation.setFunction(function.getFunctionName().toString());
22719                                                        relation.setTarget(new ResultColumnRelationshipElement(
22720                                                                        (ResultColumn) modelManager.getModel(function.getFunctionName())));
22721
22722                                                        if (relation instanceof DataFlowRelationship && option.isShowCountTableColumn()) {
22723                                                                List<TExpression> expressions = new ArrayList<TExpression>();
22724                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(), function);
22725                                                                for (int j = 0; j < expressions.size(); j++) {
22726                                                                        columnsInExpr visitor = new columnsInExpr();
22727                                                                        expressions.get(j).inOrderTraverse(visitor);
22728                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
22729                                                                        if (objectNames != null) {
22730                                                                                for (TObjectName columnName : objectNames) {
22731                                                                                        TTable tempTable = modelManager.getTable(stmt, columnName);
22732                                                                                        if (table.equals(tempTable)) {
22733                                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
22734                                                                                                                columnName, false);
22735                                                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(
22736                                                                                                                tableColumn);
22737                                                                                                relation.addSource(element);
22738                                                                                        }
22739                                                                                }
22740                                                                        }
22741                                                                }
22742                                                        }
22743                                                        if (relation.getSources().size() == 0) {
22744                                                                RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<TableRelationRows>(
22745                                                                                tableModel.getRelationRows());
22746                                                                relation.addSource(element);
22747                                                        }
22748                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
22749                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
22750                                                        AbstractRelationship relation;
22751                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
22752                                                                // relation = modelFactory.createDataFlowRelation();
22753                                                                relation = modelFactory.createRecordSetRelation();
22754                                                        } else {
22755                                                                relation = modelFactory.createRecordSetRelation();
22756                                                        }
22757                                                        relation.setEffectType(EffectType.function);
22758                                                        relation.setFunction(function.getFunctionName().toString());
22759                                                        relation.setTarget(new ResultColumnRelationshipElement(
22760                                                                        (ResultColumn) modelManager.getModel(function.getFunctionName())));
22761                                                        RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<ResultSetRelationRows>(
22762                                                                        tableModel.getRelationRows());
22763                                                        relation.addSource(element);
22764                                                }
22765                                        }
22766                                }
22767                        }
22768
22769                        if (stmt.getWhereClause() == null || stmt.getWhereClause().getCondition() == null) {
22770                                return;
22771                        }
22772
22773                        columnsInExpr visitor = new columnsInExpr();
22774                        stmt.getWhereClause().getCondition().inOrderTraverse(visitor);
22775                        List<TObjectName> objectNames = visitor.getObjectNames();
22776                        for (int j = 0; j < objectNames.size(); j++) {
22777                                TObjectName columnName = objectNames.get(j);
22778                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
22779                                        Variable tableModel;
22780                                        if (columnName.toString().indexOf(".") != -1) {
22781                                                List<String> splits = SQLUtil.parseNames(columnName.toString());
22782                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
22783                                        } else {
22784                                                tableModel = modelFactory.createVariable(columnName);
22785                                        }
22786                                        tableModel.setCreateTable(true);
22787                                        tableModel.setSubType(SubType.record);
22788                                        TObjectName variableProperties = new TObjectName();
22789                                        variableProperties.setString("*");
22790                                        modelFactory.createTableColumn(tableModel, variableProperties, true);
22791                                }
22792
22793//                              if (columnName.getColumnNameOnly().startsWith("@")
22794//                                              && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
22795//                                      continue;
22796//                              }
22797//
22798//                              if (columnName.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
22799//                                      continue;
22800//                              }
22801
22802                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
22803                                relation.setEffectType(EffectType.function);
22804                                relation.setFunction(function.getFunctionName().toString());
22805                                relation.setTarget(new ResultColumnRelationshipElement(
22806                                                (ResultColumn) modelManager.getModel(function.getFunctionName())));
22807
22808                                TTable table = modelManager.getTable(stmt, columnName);
22809                                if (table != null) {
22810                                        if (modelManager.getModel(table) instanceof Table) {
22811                                                Table tableModel = (Table) modelManager.getModel(table);
22812                                                if (tableModel != null) {
22813                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
22814                                                        if(columnModel == null) {
22815                                                                continue;
22816                                                        }
22817                                                        relation.addSource(
22818                                                                        new TableColumnRelationshipElement(columnModel, columnName.getLocation()));
22819                                                }
22820                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
22821                                                Object model = modelManager.getModel(columnName.getSourceColumn());
22822                                                if (model instanceof ResultColumn) {
22823                                                        ResultColumn resultColumn = (ResultColumn) model;
22824                                                        if (resultColumn != null) {
22825                                                                relation.addSource(
22826                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
22827                                                        }
22828                                                }
22829                                        }
22830                                }
22831                        }
22832                }
22833        }
22834
22835        private void analyzeFilterCondition(Object modelObject, TExpression expr, EJoinType joinType,
22836                        JoinClauseType joinClauseType, EffectType effectType) {
22837                if (expr == null) {
22838                        return;
22839                }
22840
22841                TCustomSqlStatement stmt = stmtStack.peek();
22842
22843                columnsInExpr visitor = new columnsInExpr();
22844                expr.inOrderTraverse(visitor);
22845
22846                List<TObjectName> objectNames = visitor.getObjectNames();
22847                List<TParseTreeNode> functions = visitor.getFunctions();
22848                List<TResultColumn> resultColumns = visitor.getResultColumns();
22849                List<TParseTreeNode> constants = visitor.getConstants();
22850
22851                ImpactRelationship relation = modelFactory.createImpactRelation();
22852                relation.setEffectType(effectType);
22853                relation.setJoinClauseType(joinClauseType);
22854                if (modelObject instanceof ResultColumn) {
22855                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
22856                } else {
22857                        ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
22858                        if (resultSet == null && stmt instanceof TUpdateSqlStatement) {
22859                                resultSet = (ResultSet) modelManager.getModel(stmt);
22860                        }
22861                        if (resultSet == null && stmt instanceof TMergeSqlStatement) {
22862                                TSelectSqlStatement subquery = ((TMergeSqlStatement) stmt).getUsingTable().getSubquery();
22863                                if (subquery != null) {
22864                                        resultSet = (ResultSet) modelManager.getModel(((TMergeSqlStatement) stmt).getUsingTable());
22865                                }
22866                                else {
22867                                        resultSet = modelFactory.createQueryTable(((TMergeSqlStatement) stmt).getUsingTable());
22868                                }
22869                        }
22870                        if (resultSet != null) {
22871                                relation.setTarget(
22872                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
22873                        }
22874                        if (stmt instanceof TDeleteSqlStatement) {
22875                                Table table = (Table) modelManager.getModel(((TDeleteSqlStatement) stmt).getTargetTable());
22876                                if (table != null) {
22877                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(table.getRelationRows()));
22878                                }
22879                        }
22880                }
22881                if (relation.getTarget() != null) {
22882
22883                        if (constants != null && constants.size() > 0) {
22884                                if (option.isShowConstantTable()) {
22885                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
22886                                        for (int i = 0; i < constants.size(); i++) {
22887                                                TParseTreeNode constant = constants.get(i);
22888                                                if (constant instanceof TConstant) {
22889                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
22890                                                                        (TConstant) constant);
22891                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
22892                                                } else if (constant instanceof TObjectName) {
22893                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
22894                                                                        (TObjectName) constant, false);
22895                                                        if(constantColumn == null) {
22896                                                                continue;
22897                                                        }
22898                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
22899                                                }
22900                                        }
22901                                }
22902                        }
22903
22904                        for (int j = 0; j < objectNames.size(); j++) {
22905                                TObjectName columnName = objectNames.get(j);
22906                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
22907                                        Variable variable = modelFactory.createVariable(columnName);
22908                                        variable.setSubType(SubType.record);
22909                                        if (variable.getColumns().isEmpty()) {
22910                                                TObjectName variableProperties = new TObjectName();
22911                                                variableProperties.setString("*");
22912                                                modelFactory.createTableColumn(variable, variableProperties, true);
22913                                        }
22914                                        relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0), columnName.getLocation()));
22915                                        continue;
22916                                }
22917
22918                                if (columnName.getColumnNameOnly().startsWith("@")
22919                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
22920                                        continue;
22921                                }
22922
22923                                if (columnName.getColumnNameOnly().startsWith(":")
22924                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
22925                                        continue;
22926                                }
22927
22928                                TTable table = modelManager.getTable(stmt, columnName);
22929
22930                                if (table == null) {
22931                                        table = columnName.getSourceTable();
22932                                }
22933
22934                                if (table == null && stmt.tables != null) {
22935                                        for (int k = 0; k < stmt.tables.size(); k++) {
22936                                                if (table != null)
22937                                                        break;
22938
22939                                                TTable tTable = stmt.tables.getTable(k);
22940                                                if (tTable.getTableType().name().startsWith("open")) {
22941                                                        continue;
22942                                                } else if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
22943                                                        for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
22944                                                                TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
22945                                                                if ("*".equals(getColumnName(refer)))
22946                                                                        continue;
22947                                                                if (getColumnName(refer).equals(getColumnName(columnName))) {
22948                                                                        table = tTable;
22949                                                                        break;
22950                                                                }
22951                                                        }
22952                                                } else if (columnName.getTableToken() != null
22953                                                                && (columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getName())
22954                                                                                || columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getAliasName()))) {
22955                                                        table = tTable;
22956                                                        break;
22957                                                }
22958                                        }
22959
22960                                        if (table == null) {
22961                                                for (int k = 0; k < stmt.tables.size(); k++) {
22962                                                        if (table != null)
22963                                                                break;
22964
22965                                                        TTable tTable = stmt.tables.getTable(k);
22966                                                        Object model = ModelBindingManager.get().getModel(tTable);
22967                                                        if (model instanceof Table) {
22968                                                                Table tableModel = (Table) model;
22969                                                                for (int z = 0; tableModel.getColumns() != null
22970                                                                                && z < tableModel.getColumns().size(); z++) {
22971                                                                        TableColumn refer = tableModel.getColumns().get(z);
22972                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
22973                                                                                table = tTable;
22974                                                                                break;
22975                                                                        }
22976                                                                        if (refer.hasStarLinkColumn()) {
22977                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
22978                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
22979                                                                                                table = tTable;
22980                                                                                                break;
22981                                                                                        }
22982                                                                                }
22983                                                                        }
22984                                                                }
22985                                                        } else if (model instanceof QueryTable) {
22986                                                                QueryTable tableModel = (QueryTable) model;
22987                                                                for (int z = 0; tableModel.getColumns() != null
22988                                                                                && z < tableModel.getColumns().size(); z++) {
22989                                                                        ResultColumn refer = tableModel.getColumns().get(z);
22990                                                                        if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
22991                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
22992                                                                                table = tTable;
22993                                                                                break;
22994                                                                        }
22995                                                                        if (refer.hasStarLinkColumn()) {
22996                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
22997                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
22998                                                                                                table = tTable;
22999                                                                                                break;
23000                                                                                        }
23001                                                                                }
23002                                                                        }
23003                                                                }
23004                                                        }
23005                                                }
23006                                        }
23007                                }
23008
23009                                if (table == null && stmt.tables != null && stmt.tables.size() != 0
23010                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
23011                                        
23012                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
23013                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
23014                                                boolean find = false;
23015                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
23016                                                        if(resultColumn.equals(modelObject)) {
23017                                                                continue;
23018                                                        }
23019                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
23020                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
23021                                                                        if (resultColumn.getColumnObject() != null) {
23022                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
23023                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
23024                                                                                if (columnName.getStartToken().posinlist >= startToken
23025                                                                                                && columnName.getEndToken().posinlist <= endToken) {
23026                                                                                        continue;
23027                                                                                }
23028                                                                        }
23029                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23030                                                                        find = true;
23031                                                                        break;
23032                                                                }
23033                                                        }
23034                                                }
23035                                                if (find) {
23036                                                        continue;
23037                                                }
23038                                        }
23039
23040                                        TObjectName pseudoTableName = new TObjectName();
23041                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
23042                                        // Otherwise fall back to default pseudo table name
23043                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
23044                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
23045                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
23046                                        pseudoTable.setPseudo(true);
23047                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
23048
23049                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
23050                                        // add the pseudo table column as source
23051                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
23052                                                relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
23053                                        }
23054
23055                                        if (isLinkOrphanColumnToFirstTable()) {
23056                                                TTable orphanTable = stmt.tables.getTable(0);
23057                                                table = stmt.tables.getTable(0);
23058                                                Object tableModel = modelManager.getModel(table);
23059                                                if (tableModel == null) {
23060                                                        tableModel = modelFactory.createTable(orphanTable);
23061                                                }
23062                                                if (tableModel instanceof Table) {
23063                                                        modelFactory.createTableColumn((Table) tableModel, columnName, false);
23064                                                        ErrorInfo errorInfo = new ErrorInfo();
23065                                                        errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
23066                                                        errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
23067                                                                        + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
23068                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
23069                                                                        columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
23070                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
23071                                                                        columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
23072                                                                        ModelBindingManager.getGlobalHash()));
23073                                                        errorInfo.fillInfo(this);
23074                                                        errorInfos.add(errorInfo);
23075                                                }
23076                                        }
23077                                }
23078
23079                                if (table != null) {
23080                                        if (modelManager.getModel(table) instanceof Table) {
23081                                                Table tableModel = (Table) modelManager.getModel(table);
23082                                                if (tableModel != null) {
23083                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
23084                                                        if(columnModel!=null) {
23085                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(columnModel,
23086                                                                                columnName.getLocation());
23087                                                                relation.addSource(element);
23088                                                        }
23089                                                }
23090                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
23091                                                QueryTable tableModel = (QueryTable)modelManager.getModel(table);
23092                                                if (table.getSubquery() != null && table.getSubquery().isCombinedQuery()) {
23093                                                        TSelectSqlStatement subquery = table.getSubquery();
23094                                                        List<ResultSet> resultSets = new ArrayList<ResultSet>();
23095                                                        if (!subquery.getLeftStmt().isCombinedQuery()) {
23096                                                                ResultSet sourceResultSet = (ResultSet) modelManager
23097                                                                                .getModel(subquery.getLeftStmt().getResultColumnList());
23098                                                                resultSets.add(sourceResultSet);
23099                                                        } else {
23100                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getLeftStmt());
23101                                                                resultSets.add(sourceResultSet);
23102                                                        }
23103
23104                                                        if (!subquery.getRightStmt().isCombinedQuery()) {
23105                                                                ResultSet sourceResultSet = (ResultSet) modelManager
23106                                                                                .getModel(subquery.getRightStmt().getResultColumnList());
23107                                                                resultSets.add(sourceResultSet);
23108                                                        } else {
23109                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getRightStmt());
23110                                                                resultSets.add(sourceResultSet);
23111                                                        }
23112
23113                                                        for (ResultSet sourceResultSet : resultSets) {
23114                                                                if (sourceResultSet != null && columnName.getSourceColumn() != null) {
23115                                                                        for (int k = 0; k < sourceResultSet.getColumns().size(); k++) {
23116                                                                                if (getColumnName(sourceResultSet.getColumns().get(k).getName()).equals(
23117                                                                                                getColumnName(columnName.getSourceColumn().getColumnNameOnly()))) {
23118                                                                                        Set<TObjectName> starLinkColumnSet = sourceResultSet.getColumns().get(k)
23119                                                                                                        .getStarLinkColumns().get(getColumnName(columnName));
23120                                                                                        if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
23121                                                                                                ResultColumn column = modelFactory.createResultColumn(sourceResultSet,
23122                                                                                                                starLinkColumnSet.iterator().next(), true);
23123                                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
23124                                                                                        } else {
23125                                                                                                relation.addSource(new ResultColumnRelationshipElement(
23126                                                                                                                sourceResultSet.getColumns().get(k)));
23127                                                                                        }
23128                                                                                }
23129                                                                        }
23130                                                                }
23131                                                        }
23132                                                } else {
23133                                                        Object model = modelManager.getModel(columnName.getSourceColumn());
23134                                                        if (model instanceof ResultColumn) {
23135                                                                ResultColumn resultColumn = (ResultColumn) model;
23136                                                                if (resultColumn != null) {
23137                                                                        if (resultColumn.hasStarLinkColumn()) {
23138                                                                                Set<TObjectName> starLinkColumnSet = resultColumn.getStarLinkColumns()
23139                                                                                                .get(getColumnName(columnName));
23140                                                                                if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
23141                                                                                        ResultColumn column = modelFactory.createResultColumn(
23142                                                                                                        resultColumn.getResultSet(), starLinkColumnSet.iterator().next(), true);
23143                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
23144                                                                                } else {
23145                                                                                        resultColumn.bindStarLinkColumn(columnName);
23146                                                                                        ResultColumn column = modelFactory
23147                                                                                                        .createResultColumn(resultColumn.getResultSet(), columnName, true);
23148                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
23149                                                                                }
23150                                                                        } else {
23151                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
23152                                                                                                resultColumn, columnName.getLocation());
23153                                                                                relation.addSource(element);
23154                                                                        }
23155                                                                }
23156                                                        }
23157                                                        else{
23158                                                                boolean find = false;
23159                                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
23160                                                                        ResultColumn resultColumn = tableModel.getColumns().get(i);
23161                                                                        if (DlineageUtil.getIdentifierNormalColumnName(resultColumn.getName()).equals(
23162                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
23163                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
23164                                                                                                resultColumn, columnName.getLocation());
23165                                                                                relation.addSource(element);
23166                                                                                find = true;
23167                                                                                break;
23168                                                                        }
23169                                                                        else if (resultColumn.getName().endsWith("*")) {
23170                                                                                resultColumn.bindStarLinkColumn(columnName);
23171                                                                        }
23172                                                                }
23173                                                                if(!find){
23174                                                                        ResultColumn resultColumn = new ResultColumn(tableModel, columnName);
23175                                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
23176                                                                                                resultColumn, columnName.getLocation());
23177                                                                                relation.addSource(element);
23178                                                                }
23179                                                        }
23180                                                }
23181                                        }
23182                                }
23183                        }
23184
23185                        for (int j = 0; j < functions.size(); j++) {
23186                                TParseTreeNode functionObj = functions.get(j);
23187                                Object functionModel = modelManager.getModel(functionObj);
23188                                if (functionModel == null) {
23189                                        functionModel = createFunction(functionObj);
23190                                }
23191                                if (functionModel instanceof Function) {
23192                                        if (functionObj instanceof TFunctionCall) {
23193                                                ResultColumn resultColumn = (ResultColumn) modelManager
23194                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
23195                                                if (resultColumn != null) {
23196                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
23197                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
23198                                                        relation.addSource(element);
23199                                                }
23200                                        }
23201                                        if (functionObj instanceof TCaseExpression) {
23202                                                ResultColumn resultColumn = (ResultColumn) modelManager
23203                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
23204                                                if (resultColumn != null) {
23205                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
23206                                                        relation.addSource(element);
23207                                                }
23208                                        }
23209                                } else if (functionModel instanceof Table) {
23210                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
23211                                                        ((TFunctionCall) functionObj).getFunctionName(), false);
23212                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
23213                                        relation.addSource(element);
23214                                }
23215                        }
23216
23217                        for (int j = 0; j < resultColumns.size(); j++) {
23218                                TResultColumn resultColumn = resultColumns.get(j);
23219                                if (modelManager.getModel(resultColumn) instanceof ResultColumn) {
23220                                        ResultColumn resultColumnModel = (ResultColumn) modelManager.getModel(resultColumn);
23221                                        relation.addSource(new ResultColumnRelationshipElement(resultColumnModel, ESqlClause.selectList));
23222                                }
23223                        }
23224                }
23225
23226                if (isShowJoin() && joinClauseType != null) {
23227                        joinInExpr joinVisitor = new joinInExpr(joinType, joinClauseType, effectType);
23228                        expr.inOrderTraverse(joinVisitor);
23229                }
23230        }
23231
23232        public void dispose() {
23233                accessedSubqueries.clear();
23234                accessedStatements.clear();
23235                stmtStack.clear();
23236                viewDDLMap.clear();
23237                procedureDDLMap.clear();
23238                structObjectMap.clear();
23239                appendResultSets.clear();
23240                appendStarColumns.clear();
23241                appendTableStarColumns.clear();
23242                modelManager.DISPLAY_ID.clear();
23243                modelManager.DISPLAY_NAME.clear();
23244                tableIds.clear();
23245                ModelBindingManager.remove();
23246        }
23247
23248        class joinTreatColumnsInExpr implements IExpressionVisitor {
23249
23250                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
23251                
23252                private TTable table;
23253
23254                public joinTreatColumnsInExpr(TTable table) {
23255                        this.table = table;
23256                }
23257
23258                public List<TObjectName> getObjectNames() {
23259                        return objectNames;
23260                }
23261
23262                boolean is_compare_condition(EExpressionType t) {
23263                        return t == EExpressionType.simple_comparison_t;
23264                }
23265
23266                @Override
23267                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
23268                        TExpression expr = (TExpression) pNode;
23269                        if (is_compare_condition(expr.getExpressionType())) {
23270                                TExpression leftExpr = expr.getLeftOperand();
23271                                columnsInExpr leftVisitor = new columnsInExpr();
23272                                leftExpr.inOrderTraverse(leftVisitor);
23273                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
23274
23275                                TExpression rightExpr = expr.getRightOperand();
23276                                columnsInExpr rightVisitor = new columnsInExpr();
23277                                rightExpr.inOrderTraverse(rightVisitor);
23278                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
23279
23280                                if (!leftObjectNames.isEmpty() && !rightObjectNames.isEmpty()) {
23281                                        for (TObjectName column : leftObjectNames) {
23282                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
23283                                                        objectNames.add(column);
23284                                                        return false;
23285                                                }
23286                                        }
23287                                        for (TObjectName column : rightObjectNames) {
23288                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
23289                                                        objectNames.add(column);
23290                                                        return false;
23291                                                }
23292                                        }
23293                                }
23294                                return false;
23295                        }
23296                        return true;
23297                }
23298        }
23299        
23300        class columnsInExpr implements IExpressionVisitor {
23301
23302                private List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
23303                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
23304                private List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
23305                private List<TResultColumn> resultColumns = new ArrayList<TResultColumn>();
23306                private List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
23307                private boolean skipFunction = false;
23308
23309                public void setSkipFunction(boolean skipFunction) {
23310                        this.skipFunction = skipFunction;
23311                }
23312
23313                public List<TParseTreeNode> getFunctions() {
23314                        return functions;
23315                }
23316
23317                public List<TSelectSqlStatement> getSubquerys() {
23318                        return subquerys;
23319                }
23320
23321                public List<TParseTreeNode> getConstants() {
23322                        return constants;
23323                }
23324
23325                public List<TObjectName> getObjectNames() {
23326                        return objectNames;
23327                }
23328
23329                public List<TResultColumn> getResultColumns() {
23330                        return resultColumns;
23331                }
23332
23333                @Override
23334                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
23335                        TExpression lcexpr = (TExpression) pNode;
23336                        // Handle named argument expressions (e.g., "INPUT => value" in Snowflake FLATTEN)
23337                        // The left operand is the parameter name, NOT a column reference.
23338                        // Only traverse the right operand (the value).
23339                        if (lcexpr.getExpressionType() == EExpressionType.assignment_t) {
23340                                // Skip left operand (parameter name) - only traverse right operand (value)
23341                                if (lcexpr.getRightOperand() != null) {
23342                                        lcexpr.getRightOperand().inOrderTraverse(this);
23343                                }
23344                                return false; // Don't continue default traversal
23345                        }
23346                        if (lcexpr.getExpressionType() == EExpressionType.simple_constant_t) {
23347                                if (lcexpr.getConstantOperand() != null) {
23348                                        if(lcexpr.getConstantOperand().getInt64_expression()!=null 
23349                                                        && lcexpr.getConstantOperand().getInt64_expression().getExpressionType() == EExpressionType.function_t) {
23350                                                lcexpr.getConstantOperand().getInt64_expression().inOrderTraverse(this);
23351                                        }
23352                                        else {
23353                                                constants.add(lcexpr.getConstantOperand());
23354                                        }
23355                                }
23356                        } else if (lcexpr.getExpressionType() == EExpressionType.array_t) {
23357                                if(lcexpr.getObjectOperand()!=null) {
23358                                        TObjectName object = lcexpr.getObjectOperand();
23359                                        objectNames.add(object);
23360                                } else if (lcexpr.getExprList() != null) {
23361                                        for (int j = 0; j < lcexpr.getExprList().size(); j++) {
23362                                                TExpression expr = lcexpr.getExprList().getExpression(j);
23363                                                if (expr != null)
23364                                                        expr.inOrderTraverse(this);
23365                                        }
23366                                }
23367                        } else if (lcexpr.getExpressionType() == EExpressionType.simple_object_name_t) {
23368                                if (lcexpr.getObjectOperand() != null && !(isBuiltInFunctionName(lcexpr.getObjectOperand())
23369                                                && isFromFunction(lcexpr.getObjectOperand()))) {
23370                                        TObjectName object = lcexpr.getObjectOperand();
23371                                        // Skip named argument parameter names (e.g., INPUT in "INPUT => value")
23372                                        // These are function parameter names, NOT column references
23373                                        if (object.getObjectType() == TObjectName.ttobjNamedArgParameter) {
23374                                                // Skip - this is a named argument parameter name
23375                                        } else if (object.getDbObjectType() == EDbObjectType.column
23376                                                        || object.getDbObjectType() == EDbObjectType.column_alias
23377                                                        || object.getDbObjectType() == EDbObjectType.alias
23378                                                        || object.getDbObjectType() == EDbObjectType.unknown
23379                                                        || object.getDbObjectType() == EDbObjectType.variable) {
23380                                                objectNames.add(object);
23381                                        } else if (object.getDbObjectType() == EDbObjectType.notAColumn
23382                                                        || object.getDbObjectType() == EDbObjectType.date_time_part ) {
23383                                                constants.add(object);
23384                                        }
23385                                }
23386                        } else if (lcexpr.getExpressionType() == EExpressionType.between_t) {
23387                                if (lcexpr.getBetweenOperand() != null && lcexpr.getBetweenOperand().getObjectOperand() != null) {
23388                                        TObjectName object = lcexpr.getBetweenOperand().getObjectOperand();
23389                                        if (object.getDbObjectType() == EDbObjectType.column
23390                                                        || object.getDbObjectType() == EDbObjectType.column_alias
23391                                                        || object.getDbObjectType() == EDbObjectType.alias
23392                                                        || object.getDbObjectType() == EDbObjectType.unknown
23393                                                        || object.getDbObjectType() == EDbObjectType.variable) {
23394                                                objectNames.add(object);
23395                                        }
23396                                }
23397                        } else if (lcexpr.getExpressionType() == EExpressionType.object_access_t) {
23398                                if (lcexpr.getObjectAccess() != null) {
23399                                        TObjectNameList objects = lcexpr.getObjectAccess().getAttributes();
23400                                        TFunctionCall function = lcexpr.getObjectAccess().getObjectExpr().getFunctionCall();
23401                                        if (objects != null && function != null) {
23402                                                for (TObjectName object : objects) {
23403                                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
23404                                                        sqlparser.sqltext = "select " + function.getFunctionName().toString() + "."
23405                                                                        + object.getColumnNameOnly() + " from " + function.getFunctionName().toString();
23406                                                        if (sqlparser.parse() == 0) {
23407                                                                TObjectName objectName = sqlparser.sqlstatements.get(0).getResultColumnList()
23408                                                                                .getResultColumn(0).getFieldAttr();
23409                                                                objectNames.add(objectName);
23410                                                        }
23411                                                }
23412                                        }
23413                                }
23414                        } else if (lcexpr.getExpressionType() == EExpressionType.function_t || lcexpr.getExpressionType() == EExpressionType.fieldselection_t) {
23415                                TFunctionCall func = lcexpr.getFunctionCall();
23416                                if (func == null) {
23417                                        return true;
23418                                }
23419                                if (skipFunction) {
23420                                        if (func.getArgs() != null) {
23421                                                for (int k = 0; k < func.getArgs().size(); k++) {
23422                                                        TExpression expr = func.getArgs().getExpression(k);
23423                                                        if (expr != null)
23424                                                                expr.inOrderTraverse(this);
23425                                                }
23426                                        }
23427
23428                                        if (func.getTrimArgument() != null) {
23429                                                TTrimArgument args = func.getTrimArgument();
23430                                                TExpression expr = args.getStringExpression();
23431                                                if (expr != null) {
23432                                                        expr.inOrderTraverse(this);
23433                                                }
23434                                                expr = args.getTrimCharacter();
23435                                                if (expr != null) {
23436                                                        expr.inOrderTraverse(this);
23437                                                }
23438                                        }
23439
23440                                        if (func.getAgainstExpr() != null) {
23441                                                func.getAgainstExpr().inOrderTraverse(this);
23442                                        }
23443//                                      if (func.getBetweenExpr() != null) {
23444//                                              func.getBetweenExpr().inOrderTraverse(this);
23445//                                      }
23446                                        if (func.getExpr1() != null) {
23447                                                func.getExpr1().inOrderTraverse(this);
23448                                        }
23449                                        if (func.getExpr2() != null) {
23450                                                func.getExpr2().inOrderTraverse(this);
23451                                        }
23452                                        if (func.getExpr3() != null) {
23453                                                func.getExpr3().inOrderTraverse(this);
23454                                        }
23455                                        if (func.getParameter() != null) {
23456                                                func.getParameter().inOrderTraverse(this);
23457                                        }
23458                                } else {
23459                                        functions.add(func);
23460                                }
23461
23462                        } else if (lcexpr.getExpressionType() == EExpressionType.case_t) {
23463                                TCaseExpression expr = lcexpr.getCaseExpression();
23464                                if (skipFunction) {
23465                                        TExpression defaultExpr = expr.getElse_expr();
23466                                        if (defaultExpr != null) {
23467                                                defaultExpr.inOrderTraverse(this);
23468                                        }
23469                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
23470                                        for (int i = 0; i < list.size(); i++) {
23471                                                TWhenClauseItem element = (TWhenClauseItem) list.getElement(i);
23472                                                (((TWhenClauseItem) element).getReturn_expr()).inOrderTraverse(this);
23473
23474                                        }
23475                                } else {
23476                                        functions.add(expr);
23477                                }
23478                        } else if (lcexpr.getSubQuery() != null) {
23479                                TSelectSqlStatement select = lcexpr.getSubQuery();
23480                                analyzeSelectStmt(select);
23481                                subquerys.add(select);
23482                                if (select.getResultColumnList() != null && select.getResultColumnList().size() > 0) {
23483                                        for (TResultColumn column : select.getResultColumnList()) {
23484                                                resultColumns.add(column);
23485                                        }
23486                                }
23487                        }
23488                        return true;
23489                }
23490        }
23491
23492        class joinInExpr implements IExpressionVisitor {
23493
23494                private EJoinType joinType;
23495                private JoinClauseType joinClauseType;
23496                private EffectType effectType;
23497
23498                public joinInExpr(EJoinType joinType, JoinClauseType joinClauseType, EffectType effectType) {
23499                        this.joinType = joinType;
23500                        this.joinClauseType = joinClauseType;
23501                        this.effectType = effectType;
23502                }
23503
23504                boolean is_compare_condition(EExpressionType t) {
23505                        return ((t == EExpressionType.simple_comparison_t) || (t == EExpressionType.group_comparison_t)
23506                                        || (t == EExpressionType.in_t) || (t == EExpressionType.pattern_matching_t)
23507                                        || (t == EExpressionType.left_join_t) || (t == EExpressionType.right_join_t));
23508                }
23509
23510                @Override
23511                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
23512                        TExpression expr = (TExpression) pNode;
23513                        if (is_compare_condition(expr.getExpressionType())) {
23514                                TExpression leftExpr = expr.getLeftOperand();
23515                                columnsInExpr leftVisitor = new columnsInExpr();
23516                                leftExpr.inOrderTraverse(leftVisitor);
23517                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
23518                                List<TParseTreeNode> leftObjects = leftVisitor.getFunctions();
23519                                leftObjects.addAll(leftObjectNames);
23520                                
23521                                TExpression rightExpr = expr.getRightOperand();
23522                                columnsInExpr rightVisitor = new columnsInExpr();
23523                                rightExpr.inOrderTraverse(rightVisitor);
23524                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
23525                                List<TParseTreeNode> rightObjects = rightVisitor.getFunctions();
23526                                rightObjects.addAll(rightObjectNames);
23527
23528                                if (!leftObjects.isEmpty() && !rightObjects.isEmpty()) {
23529                                        TCustomSqlStatement stmt = stmtStack.peek();
23530
23531                                        for (int i = 0; i < leftObjects.size(); i++) {
23532                                                TParseTreeNode leftObject = leftObjects.get(i);
23533                                                TTable leftTable = null;
23534                                                TFunctionCall leftFunction = null;
23535                                                TObjectName leftObjectName = null;
23536                                                if (leftObject instanceof TObjectName) {
23537                                                        leftObjectName = (TObjectName)leftObject;
23538
23539                                                        if (leftObjectName.getDbObjectType() == EDbObjectType.variable) {
23540                                                                continue;
23541                                                        }
23542
23543                                                        if (leftObjectName.getColumnNameOnly().startsWith("@")
23544                                                                        && (option.getVendor() == EDbVendor.dbvmssql
23545                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
23546                                                                continue;
23547                                                        }
23548
23549                                                        if (leftObjectName.getColumnNameOnly().startsWith(":")
23550                                                                        && (option.getVendor() == EDbVendor.dbvhana
23551                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
23552                                                                continue;
23553                                                        }
23554
23555                                                        leftTable = modelManager.getTable(stmt, leftObjectName);
23556
23557                                                        if (leftTable == null) {
23558                                                                leftTable = leftObjectName.getSourceTable();
23559                                                        }
23560
23561                                                        if (leftTable == null) {
23562                                                                leftTable = modelManager.guessTable(stmt, leftObjectName);
23563                                                        }
23564                                                }
23565                                                else if(leftObject instanceof TFunctionCall){
23566                                                        leftFunction = (TFunctionCall)leftObject;
23567                                                }
23568
23569                                                if (leftTable != null || leftFunction != null) {
23570                                                        for (int j = 0; j < rightObjects.size(); j++) {
23571                                                                JoinRelationship joinRelation = modelFactory.createJoinRelation();
23572                                                                joinRelation.setEffectType(effectType);
23573                                                                if (joinType != null) {
23574                                                                        joinRelation.setJoinType(joinType);
23575                                                                } else {
23576                                                                        if (expr.getLeftOperand().isOracleOuterJoin()) {
23577                                                                                joinRelation.setJoinType(right);
23578                                                                        } else if (expr.getRightOperand().isOracleOuterJoin()) {
23579                                                                                joinRelation.setJoinType(EJoinType.left);
23580                                                                        } else if (expr.getExpressionType() == EExpressionType.left_join_t) {
23581                                                                                joinRelation.setJoinType(EJoinType.left);
23582                                                                        } else if (expr.getExpressionType() == EExpressionType.right_join_t) {
23583                                                                                joinRelation.setJoinType(right);
23584                                                                        } else {
23585                                                                                joinRelation.setJoinType(EJoinType.inner);
23586                                                                        }
23587                                                                }
23588
23589                                                                joinRelation.setJoinClauseType(joinClauseType);
23590                                                                joinRelation.setJoinCondition(expr.toString());
23591
23592
23593                                                                if (leftTable != null) {
23594                                                                        if (modelManager.getModel(leftTable) instanceof Table) {
23595                                                                                Table tableModel = (Table) modelManager.getModel(leftTable);
23596                                                                                if (tableModel != null) {
23597                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
23598                                                                                                        leftObjectName, false);
23599                                                                                        if (columnModel != null) {
23600                                                                                                joinRelation.addSource(new TableColumnRelationshipElement(columnModel));
23601                                                                                        }
23602                                                                                }
23603                                                                        } else if (modelManager.getModel(leftTable) instanceof QueryTable) {
23604                                                                                QueryTable table = (QueryTable) modelManager.getModel(leftTable);
23605                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
23606                                                                                if (subquery != null && subquery.isCombinedQuery()) {
23607                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
23608                                                                                                        leftObjectName);
23609                                                                                        if (resultColumn != null) {
23610                                                                                                joinRelation
23611                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
23612                                                                                        }
23613                                                                                } else if (leftObjectName.getSourceColumn() != null) {
23614                                                                                        Object model = modelManager.getModel(leftObjectName);
23615                                                                                        if (model == null) {
23616                                                                                                model = modelFactory.createResultColumn(table, leftObjectName);
23617                                                                                        }
23618                                                                                        if (model instanceof ResultColumn) {
23619                                                                                                ResultColumn resultColumn = (ResultColumn) model;
23620                                                                                                if (resultColumn != null) {
23621                                                                                                        joinRelation.addSource(
23622                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
23623                                                                                                }
23624                                                                                        } else if (model instanceof LinkedHashMap) {
23625                                                                                                String columnName = getColumnNameOnly(leftObjectName.toString());
23626                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
23627                                                                                                if (resultColumns.containsKey(columnName)) {
23628                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
23629                                                                                                        joinRelation.addSource(
23630                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
23631                                                                                                }
23632                                                                                        }
23633                                                                                } else {
23634                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
23635                                                                                                        leftObjectName);
23636                                                                                        if (resultColumn != null) {
23637                                                                                                joinRelation
23638                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
23639                                                                                        }
23640                                                                                }
23641                                                                        }
23642                                                                }
23643                                                                else if(leftFunction!=null) {
23644                                                                        Object functionObj = createFunction(leftFunction);
23645                                                                        if(functionObj instanceof Function) {
23646                                                                                Function function = (Function)functionObj;
23647                                                                                joinRelation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
23648                                                                        }
23649                                                                }
23650
23651                                                                TParseTreeNode rightObject = rightObjects.get(j);
23652                                                                if(rightObject instanceof TObjectName) {
23653                                                                        TObjectName rightObjectName = (TObjectName)rightObject;
23654
23655                                                                        if (rightObjectName.getDbObjectType() == EDbObjectType.variable) {
23656                                                                                continue;
23657                                                                        }
23658
23659                                                                        if (rightObjectName.getColumnNameOnly().startsWith("@")
23660                                                                                        && (option.getVendor() == EDbVendor.dbvmssql
23661                                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
23662                                                                                continue;
23663                                                                        }
23664
23665                                                                        if (rightObjectName.getColumnNameOnly().startsWith(":")
23666                                                                                        && (option.getVendor() == EDbVendor.dbvhana
23667                                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
23668                                                                                continue;
23669                                                                        }
23670
23671                                                                        TTable rightTable = modelManager.getTable(stmt, rightObjectName);
23672                                                                        if (rightTable == null) {
23673                                                                                rightTable = rightObjectName.getSourceTable();
23674                                                                        }
23675
23676                                                                        if (rightTable == null) {
23677                                                                                rightTable = modelManager.guessTable(stmt, rightObjectName);
23678                                                                        }
23679
23680                                                                        if (modelManager.getModel(rightTable) instanceof Table) {
23681                                                                                Table tableModel = (Table) modelManager.getModel(rightTable);
23682                                                                                if (tableModel != null) {
23683                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
23684                                                                                                        rightObjectName, false);
23685                                                                                        if(columnModel != null) {
23686                                                                                                joinRelation.setTarget(new TableColumnRelationshipElement(columnModel));
23687                                                                                        }
23688                                                                                }
23689                                                                        } else if (modelManager.getModel(rightTable) instanceof QueryTable) {
23690                                                                                QueryTable table = (QueryTable) modelManager.getModel(rightTable);
23691                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
23692                                                                                if (subquery != null && subquery.isCombinedQuery()) {
23693                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
23694                                                                                                        rightObjectName);
23695                                                                                        if (resultColumn != null) {
23696                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23697                                                                                        }
23698                                                                                } else if (rightObjectName.getSourceColumn() != null) {
23699                                                                                        Object model = modelManager.getModel(rightObjectName);
23700                                                                                        if (model == null) {
23701                                                                                                model = modelManager
23702                                                                                                                .getModel(rightObjectName.getSourceColumn());
23703                                                                                        }
23704                                                                                        if (model instanceof ResultColumn) {
23705                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement((ResultColumn)model));
23706                                                                                        }
23707                                                                                        else if (model instanceof LinkedHashMap) {
23708                                                                                                String columnName = getColumnNameOnly(rightObjectName.toString());
23709                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
23710                                                                                                if (resultColumns.containsKey(columnName)) {
23711                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
23712                                                                                                        joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23713                                                                                                }
23714                                                                                        }
23715                                                                                } else {
23716                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
23717                                                                                                        rightObjectName);
23718                                                                                        if (resultColumn != null) {
23719                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23720                                                                                        }
23721                                                                                }
23722                                                                        }
23723                                                                }
23724                                                                else if(rightObject instanceof TFunctionCall) {
23725                                                                        Object functionObj = createFunction(rightObject);
23726                                                                        if(functionObj instanceof Function) {
23727                                                                                Function function = (Function)functionObj;
23728                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(function.getColumns().get(0)));
23729                                                                        }
23730                                                                }
23731                                                        }
23732                                                }
23733                                        }
23734                                }
23735                        }
23736                        return true;
23737                }
23738        }
23739
23740        @Deprecated
23741        public static Dataflow getSqlflowJSONModel(dataflow dataflow) {
23742                EDbVendor vendor = ModelBindingManager.getGlobalVendor();
23743                if(vendor == null) {
23744                        throw new IllegalArgumentException("getSqlflowJSONModel(dataflow dataflow) is deprecated, please call method getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor).");
23745                }
23746                return getSqlflowJSONModel(vendor, dataflow, false);
23747        }
23748
23749        public static Dataflow getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor) {
23750                return getSqlflowJSONModel(vendor, dataflow, false);
23751        }
23752
23753        public static Dataflow getSqlflowJSONModel(EDbVendor vendor, dataflow dataflow, boolean normalizeIdentifier) {
23754                Dataflow model = new Dataflow();
23755                
23756                if (dataflow.getErrors() != null && !dataflow.getErrors().isEmpty()) {
23757                        List<Error> errorList = new ArrayList<Error>();
23758                        for (error error : dataflow.getErrors()) {
23759                                Error err = new Error();
23760                                err.setErrorMessage(error.getErrorMessage());
23761                                err.setErrorType(error.getErrorType());
23762                                err.setCoordinates(Coordinate.parse(error.getCoordinate()));
23763                                err.setFile(err.getFile());
23764                                err.setOriginCoordinates(Coordinate.parse(error.getOriginCoordinate()));
23765                                errorList.add(err);
23766                        }
23767                        model.setErrors(errorList.toArray(new Error[0]));
23768                }
23769
23770                Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(vendor, dataflow);
23771                sqlflow.setErrorMessages(null);
23772                model.setDbobjs(sqlflow);
23773                model.setOrientation(dataflow.getOrientation());
23774
23775
23776                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process> processes = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process>();
23777                if(dataflow.getProcesses()!=null){
23778                        for(process process: dataflow.getProcesses()){
23779                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Process processModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process();
23780                                processModel.setId(process.getId());
23781                                processModel.setName(process.getName());
23782                                processModel.setProcedureId(process.getProcedureId());
23783                                processModel.setProcedureName(process.getProcedureName());
23784                                processModel.setType(process.getType());
23785                                processModel.setCoordinate(process.getCoordinate());
23786                                processModel.setDatabase(process.getDatabase());
23787                                processModel.setSchema(process.getSchema());
23788                                processModel.setServer(process.getServer());
23789                                processModel.setQueryHashId(process.getQueryHashId());
23790                                if (process.getTransforms() != null && !process.getTransforms().isEmpty()) {
23791                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
23792                                        for (transform transform : process.getTransforms()) {
23793                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
23794                                                item.setCode(transform.getCode());
23795                                                item.setType(transform.getType());
23796                                                item.setCoordinate(transform.getCoordinate(true));
23797                                                transforms.add(item);
23798                                        }
23799                                        processModel.setTransforms(transforms
23800                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
23801                                }
23802                                processes.add(processModel);
23803                        }
23804                }
23805                model.setProcesses(processes.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process[0]));
23806
23807                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship> relations = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship>();
23808                if (dataflow.getRelationships() != null) {
23809                        for (relationship relation : dataflow.getRelationships()) {
23810                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship relationModel;
23811                                if (relation.getType().equals("join")) {
23812                                        gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship joinRelationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship();
23813                                        joinRelationModel.setCondition(relation.getCondition());
23814                                        joinRelationModel.setJoinType(relation.getJoinType());
23815                                        joinRelationModel.setClause(relation.getClause());
23816                                        relationModel = joinRelationModel;
23817                                } else {
23818                                        relationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship();
23819                                }
23820
23821                                relationModel.setId(relation.getId());
23822                                relationModel.setProcessId(relation.getProcessId());
23823                                relationModel.setProcessType(relation.getProcessType());
23824                                relationModel.setType(relation.getType());
23825                                relationModel.setEffectType(relation.getEffectType());
23826                                relationModel.setPartition(relation.getPartition());
23827                                relationModel.setFunction(relation.getFunction());
23828                                relationModel.setProcedureId(relation.getProcedureId());
23829                                relationModel.setSqlHash(relation.getSqlHash());
23830                                relationModel.setCondition(relation.getCondition());
23831                                relationModel.setSqlComment(relation.getSqlComment());
23832                                relationModel.setTimestampMax(relation.getTimestampMax());
23833                                relationModel.setTimestampMin(relation.getTimestampMin());
23834                                if (Boolean.TRUE.equals(relation.getBuiltIn())) {
23835                                        relationModel.setBuiltIn(relation.getBuiltIn());
23836                                }
23837                                relationModel.setCallStmt(relation.getCallStmt());
23838                                relationModel.setCallCoordinate(relation.getCallCoordinate());
23839                                
23840                                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
23841                                        {
23842                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
23843                                                targetColumn target = relation.getTarget();
23844                                                if (normalizeIdentifier) {
23845                                                        targetModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, target.getColumn()));
23846                                                        targetModel.setParentName(
23847                                                                        SQLUtil.getIdentifierNormalTableName(vendor, target.getParent_name()));
23848                                                        targetModel.setTargetName(
23849                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, target.getTarget_name()));
23850                                                } else {
23851                                                        targetModel.setColumn(target.getColumn());
23852                                                        targetModel.setParentName(target.getParent_name());
23853                                                        targetModel.setTargetName(target.getTarget_name());
23854                                                }
23855                                                targetModel.setId(target.getId());
23856                                                targetModel.setTargetId(target.getTarget_id());
23857                                                targetModel.setParentId(target.getParent_id());
23858                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
23859                                                targetModel.setFunction(target.getFunction());
23860                                                targetModel.setType(target.getType());
23861                                                relationModel.setTarget(targetModel);
23862                                        }
23863
23864                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
23865                                        for (sourceColumn source : relation.getSources()) {
23866                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
23867                                                if (normalizeIdentifier) {
23868                                                        sourceModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, source.getColumn()));
23869                                                        sourceModel.setParentName(
23870                                                                        SQLUtil.getIdentifierNormalTableName(vendor, source.getParent_name()));
23871                                                        sourceModel.setSourceName(
23872                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, source.getSource_name()));
23873                                                } else {
23874                                                        sourceModel.setColumn(source.getColumn());
23875                                                        sourceModel.setParentName(source.getParent_name());
23876                                                        sourceModel.setSourceName(source.getSource_name());
23877                                                }
23878                                                sourceModel.setColumnType(source.getColumn_type());
23879                                                sourceModel.setId(source.getId());
23880                                                sourceModel.setParentId(source.getParent_id());
23881                                                sourceModel.setSourceId(source.getSource_id());
23882                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
23883                                                sourceModel.setClauseType(source.getClauseType());
23884                                                sourceModel.setType(source.getType());
23885                                                sourceModels.add(sourceModel);
23886                                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
23887                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
23888                                                        for (transform transform : source.getTransforms()) {
23889                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
23890                                                                item.setCode(transform.getCode());
23891                                                                item.setType(transform.getType());
23892                                                                item.setCoordinate(transform.getCoordinate(true));
23893                                                                transforms.add(item);
23894                                                        }
23895                                                        sourceModel.setTransforms(transforms
23896                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
23897                                                }
23898                                                
23899                                                if (source.getCandidateParents() != null && !source.getCandidateParents().isEmpty()) {
23900                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable> candidateParents = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable>();
23901                                                        for (candidateTable candidateTable : source.getCandidateParents()) {
23902                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable();
23903                                                                item.setId(candidateTable.getId());
23904                                                                if (normalizeIdentifier) {
23905                                                                        item.setName(
23906                                                                                        SQLUtil.getIdentifierNormalTableName(vendor, candidateTable.getName()));
23907                                                                } else {
23908                                                                        item.setName(candidateTable.getName());
23909                                                                }
23910                                                                candidateParents.add(item);
23911                                                        }
23912                                                        sourceModel.setCandidateParents(candidateParents
23913                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable[0]));
23914                                                }
23915                                        }
23916                                        relationModel.setSources(sourceModels
23917                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
23918                                        relations.add(relationModel);
23919                                } else if (relation.getCaller() != null && relation.getCallees() != null
23920                                                && !relation.getCallees().isEmpty()) {
23921                                        {
23922                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
23923                                                targetColumn target = relation.getCaller();
23924                                                if (normalizeIdentifier) {
23925                                                        targetModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, target.getName()));
23926                                                } else {
23927                                                        targetModel.setName(target.getName());
23928                                                }
23929                                                targetModel.setId(target.getId());
23930                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
23931                                                targetModel.setType(target.getType());
23932                                                relationModel.setCaller(targetModel);
23933                                        }
23934
23935                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
23936                                        for (sourceColumn source : relation.getCallees()) {
23937                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
23938                                                if (normalizeIdentifier) {
23939                                                        sourceModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, source.getName()));
23940                                                } else {
23941                                                        sourceModel.setName(source.getName());
23942                                                }
23943                                                sourceModel.setId(source.getId());
23944                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
23945                                                sourceModel.setType(source.getType());
23946                                                sourceModels.add(sourceModel);
23947                                        }
23948                                        relationModel.setCallees(sourceModels
23949                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
23950                                        relations.add(relationModel);
23951                                }
23952                        }
23953                }
23954                model.setRelationships(relations.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship[0]));
23955                return model;
23956        }
23957
23958        public static String getVersion() {
23959                return "3.1.4";
23960        }
23961
23962        public static String getReleaseDate() {
23963                return "2023-03-04";
23964        }
23965
23966        public static void main(String[] args) {
23967                if (args.length < 1) {
23968                        System.out.println(
23969                                        "Usage: java DataFlowAnalyzer [/f <path_to_sql_file>] [/d <path_to_directory_includes_sql_files>] [/s [/text]] [/json] [/traceView] [/t <database type>] [/o <output file path>][/version]");
23970                        System.out.println("/f: Option, specify the sql file path to analyze fdd relation.");
23971                        System.out.println("/d: Option, specify the sql directory path to analyze fdd relation.");
23972                        System.out.println("/j: Option, analyze the join relation.");
23973                        System.out.println("/s: Option, simple output, ignore the intermediate results.");
23974                        System.out.println("/i: Option, ignore all result sets.");
23975                        System.out.println("/traceView: Option, analyze the source tables of views.");
23976                        System.out.println("/text: Option, print the plain text format output.");
23977                        System.out.println("/json: Option, print the json format output.");
23978                        System.out.println(
23979                                        "/t: Option, set the database type. Support oracle, mysql, mssql, db2, netezza, teradata, informix, sybase, postgresql, hive, greenplum and redshift, the default type is oracle");
23980                        System.out.println("/o: Option, write the output stream to the specified file.");
23981                        System.out.println("/log: Option, generate a dataflow.log file to log information.");
23982                        return;
23983                }
23984
23985                File sqlFiles = null;
23986
23987                List<String> argList = Arrays.asList(args);
23988
23989                if (argList.indexOf("/version") != -1) {
23990                        System.out.println("Version: " + DataFlowAnalyzer.getVersion());
23991                        System.out.println("Release Date: " + DataFlowAnalyzer.getReleaseDate());
23992                        return;
23993                }
23994
23995                if (argList.indexOf("/f") != -1 && argList.size() > argList.indexOf("/f") + 1) {
23996                        sqlFiles = new File(args[argList.indexOf("/f") + 1]);
23997                        if (!sqlFiles.exists() || !sqlFiles.isFile()) {
23998                                System.out.println(sqlFiles + " is not a valid file.");
23999                                return;
24000                        }
24001                } else if (argList.indexOf("/d") != -1 && argList.size() > argList.indexOf("/d") + 1) {
24002                        sqlFiles = new File(args[argList.indexOf("/d") + 1]);
24003                        if (!sqlFiles.exists() || !sqlFiles.isDirectory()) {
24004                                System.out.println(sqlFiles + " is not a valid directory.");
24005                                return;
24006                        }
24007                } else {
24008                        System.out.println("Please specify a sql file path or directory path to analyze dlineage.");
24009                        return;
24010                }
24011
24012                EDbVendor vendor = EDbVendor.dbvoracle;
24013
24014                int index = argList.indexOf("/t");
24015
24016                if (index != -1 && args.length > index + 1) {
24017                        vendor = TGSqlParser.getDBVendorByName(args[index + 1]);
24018                }
24019
24020                String outputFile = null;
24021
24022                index = argList.indexOf("/o");
24023
24024                if (index != -1 && args.length > index + 1) {
24025                        outputFile = args[index + 1];
24026                }
24027
24028                FileOutputStream writer = null;
24029                if (outputFile != null) {
24030                        try {
24031                                writer = new FileOutputStream(outputFile);
24032                                System.setOut(new PrintStream(writer));
24033                        } catch (FileNotFoundException e) {
24034                                logger.error("output file is not found.", e);
24035                        }
24036                }
24037
24038                boolean simple = argList.indexOf("/s") != -1;
24039                boolean ignoreResultSets = argList.indexOf("/i") != -1;
24040                boolean showJoin = argList.indexOf("/j") != -1;
24041                boolean textFormat = false;
24042                boolean jsonFormat = false;
24043                if (simple) {
24044                        textFormat = argList.indexOf("/text") != -1;
24045                }
24046
24047                boolean traceView = argList.indexOf("/traceView") != -1;
24048                if (traceView) {
24049                        simple = true;
24050                }
24051
24052                jsonFormat = argList.indexOf("/json") != -1;
24053
24054                DataFlowAnalyzer dlineage = new DataFlowAnalyzer(sqlFiles, vendor, simple);
24055
24056                dlineage.setShowJoin(showJoin);
24057                dlineage.setIgnoreRecordSet(ignoreResultSets);
24058                // dlineage.setShowImplicitSchema(true);
24059
24060                if (simple && !jsonFormat) {
24061                        dlineage.setTextFormat(textFormat);
24062                }
24063
24064                String result = dlineage.generateDataFlow();
24065
24066//              dataflow dataflow = ProcessUtility.generateTableLevelLineage(dlineage, dlineage.getDataFlow());
24067//              System.out.println(result);
24068
24069                if (jsonFormat) {
24070                        // Map jsonResult = new LinkedHashMap();
24071                        Dataflow model = getSqlflowJSONModel(vendor, dlineage.getDataFlow(), true);
24072                        // jsonResult.put("data", BeanUtils.bean2Map(model));
24073                        result = JSON.toJSONString(model);
24074                } else if (traceView) {
24075                        result = dlineage.traceView();
24076                }
24077
24078                if (result != null) {
24079                        System.out.println(result);
24080
24081                        if (writer != null && result.length() < 1024 * 1024) {
24082                                System.err.println(result);
24083                        }
24084                }
24085
24086                try {
24087                        if (writer != null) {
24088                                writer.close();
24089                        }
24090                } catch (IOException e) {
24091                        logger.error("close writer failed.", e);
24092                }
24093
24094                boolean log = argList.indexOf("/log") != -1;
24095
24096                PrintStream systemSteam = System.err;
24097                ByteArrayOutputStream sw = new ByteArrayOutputStream();
24098                PrintStream pw = new PrintStream(sw);
24099                System.setErr(pw);
24100
24101
24102                List<ErrorInfo> errors = dlineage.getErrorMessages();
24103                if (!errors.isEmpty()) {
24104                        System.err.println("Error log:\n");
24105                        for (int i = 0; i < errors.size(); i++) {
24106                                System.err.println(errors.get(i).getErrorMessage());
24107                        }
24108                }
24109
24110                if (sw != null) {
24111                        String errorMessage = sw.toString().trim();
24112                        if (errorMessage.length() > 0) {
24113                                if (log) {
24114                                        try {
24115                                                pw = new PrintStream(new File(".", "dataflow.log"));
24116                                                pw.print(errorMessage);
24117                                        } catch (FileNotFoundException e) {
24118                                                logger.error("error log file is not found.", e);
24119                                        }
24120                                }
24121
24122                                System.setErr(systemSteam);
24123                                System.err.println(errorMessage);
24124                        }
24125                }
24126        }
24127
24128        public List<ErrorInfo> getErrorMessages() {
24129                return errorInfos;
24130        }
24131
24132        public String traceView() {
24133                StringBuilder buffer = new StringBuilder();
24134                dataflow dataflow = this.getDataFlow();
24135                Map<table, Set<table>> traceViewMap = new LinkedHashMap<table, Set<table>>();
24136                if (dataflow != null && dataflow.getViews() != null) {
24137                        List<relationship> relations = dataflow.getRelationships();
24138                        Map<String, table> viewMap = new HashMap<String, table>();
24139                        Map<String, table> tableMap = new HashMap<String, table>();
24140                        for (table view : dataflow.getViews()) {
24141                                viewMap.put(view.getId(), view);
24142                                tableMap.put(view.getId(), view);
24143                        }
24144                        for (table table : dataflow.getTables()) {
24145                                tableMap.put(table.getId(), table);
24146                        }
24147                        for (relationship relation : relations) {
24148                                if (!RelationshipType.fdd.name().equals(relation.getType())) {
24149                                        continue;
24150                                }
24151                                String parentId = relation.getTarget().getParent_id();
24152                                if (viewMap.containsKey(parentId)) {
24153                                        if (!traceViewMap.containsKey(viewMap.get(parentId))) {
24154                                                traceViewMap.put(viewMap.get(parentId), new LinkedHashSet<table>());
24155                                        }
24156
24157                                        for (sourceColumn sourceColumn : relation.getSources()) {
24158                                                traceViewMap.get(viewMap.get(parentId)).add(tableMap.get(sourceColumn.getParent_id()));
24159                                        }
24160                                }
24161                        }
24162
24163                        Map<table, Set<table>> viewTableMap = new LinkedHashMap<table, Set<table>>();
24164                        for (table view : traceViewMap.keySet()) {
24165                                Set<table> tables = new LinkedHashSet<table>();
24166                                traverseViewSourceTables(tables, view, traceViewMap);
24167                                viewTableMap.put(view, tables);
24168                        }
24169
24170                        for (table view : viewTableMap.keySet()) {
24171                                buffer.append(view.getFullName());
24172                                for (table table : viewTableMap.get(view)) {
24173                                        buffer.append(",").append(table.getFullName());
24174                                }
24175                                buffer.append(System.getProperty("line.separator"));
24176                        }
24177                }
24178                return buffer.toString().trim();
24179        }
24180
24181        private void traverseViewSourceTables(Set<table> tables, table view, Map<table, Set<table>> traceViewMap) {
24182                Set<table> sourceTables = traceViewMap.get(view);
24183                for (table sourceTable : sourceTables) {
24184                        if (sourceTable.isTable()) {
24185                                tables.add(sourceTable);
24186                        } else if (sourceTable.isView()) {
24187                                traverseViewSourceTables(tables, sourceTable, traceViewMap);
24188                        }
24189                }
24190        }
24191
24192        protected List<SqlInfo> convertSQL(EDbVendor vendor, String json) {
24193                List<SqlInfo> sqlInfos = new ArrayList<SqlInfo>();
24194                List sqlContents = (List) JSON.parseObject(json);
24195                for (int j = 0; j < sqlContents.size(); j++) {
24196                        Map sqlContent = (Map) sqlContents.get(j);
24197                        String sql = (String) sqlContent.get("sql");
24198                        String fileName = (String) sqlContent.get("fileName");
24199                        String filePath = (String) sqlContent.get("filePath");
24200                        if (sql != null && sql.trim().startsWith("{")) {
24201                                if (sql.indexOf("createdBy") != -1) {
24202                                        if (this.sqlenv == null) {
24203                                                TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(),
24204                                                                option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql);
24205                                                if (sqlenvs != null && sqlenvs.length > 0) {
24206                                                        this.sqlenv = sqlenvs[0];
24207                                                }
24208                                        }
24209                                        if (sql.toLowerCase().indexOf("sqldep") != -1 || sql.toLowerCase().indexOf("grabit") != -1) {
24210                                                Map queryObject = (Map) JSON.parseObject(sql);
24211                                                List querys = (List) queryObject.get("queries");
24212                                                if (querys != null) {
24213                                                        for (int i = 0; i < querys.size(); i++) {
24214                                                                Map object = (Map) querys.get(i);
24215                                                                SqlInfo info = new SqlInfo();
24216                                                                info.setSql(JSON.toJSONString(object));
24217                                                                info.setFileName(fileName);
24218                                                                info.setFilePath(filePath);
24219                                                                info.setOriginIndex(i);
24220                                                                sqlInfos.add(info);
24221                                                        }
24222                                                        queryObject.remove("queries");
24223                                                        SqlInfo info = new SqlInfo();
24224                                                        info.setSql(JSON.toJSONString(queryObject));
24225                                                        info.setFileName(fileName);
24226                                                        info.setFilePath(filePath);
24227                                                        info.setOriginIndex(querys.size());
24228                                                        sqlInfos.add(info);
24229                                                } else {
24230                                                        SqlInfo info = new SqlInfo();
24231                                                        info.setSql(JSON.toJSONString(queryObject));
24232                                                        info.setFileName(fileName);
24233                                                        info.setFilePath(filePath);
24234                                                        info.setOriginIndex(0);
24235                                                        sqlInfos.add(info);
24236                                                }
24237                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
24238                                                Map sqlflow = (Map) JSON.parseObject(sql);
24239                                                List<Map> servers = (List<Map>) sqlflow.get("servers");
24240                                                if (servers != null) {
24241                                                        for (Map queryObject : servers) {
24242                                                                String name = (String) queryObject.get("name");
24243                                                                String dbVendor = (String) queryObject.get("dbVendor");
24244                                                                List querys = (List) queryObject.get("queries");
24245                                                                if (querys != null) {
24246                                                                        for (int i = 0; i < querys.size(); i++) {
24247                                                                                Map object = (Map) querys.get(i);
24248                                                                                SqlInfo info = new SqlInfo();
24249                                                                                info.setSql(JSON.toJSONString(object));
24250                                                                                info.setFileName(fileName);
24251                                                                                info.setFilePath(filePath);
24252                                                                                info.setOriginIndex(i);
24253                                                                                info.setDbVendor(dbVendor);
24254                                                                                info.setServer(name);
24255                                                                                sqlInfos.add(info);
24256                                                                        }
24257                                                                        queryObject.remove("queries");
24258                                                                        Map serverObject = new IndexedLinkedHashMap();
24259                                                                        serverObject.put("createdBy", sqlflow.get("createdBy"));
24260                                                                        serverObject.put("servers", Arrays.asList(queryObject));
24261                                                                        SqlInfo info = new SqlInfo();
24262                                                                        info.setSql(JSON.toJSONString(serverObject));
24263                                                                        info.setFileName(fileName);
24264                                                                        info.setFilePath(filePath);
24265                                                                        info.setOriginIndex(querys.size());
24266                                                                        info.setDbVendor(dbVendor);
24267                                                                        info.setServer(filePath);
24268                                                                        sqlInfos.add(info);
24269                                                                } else {
24270                                                                        SqlInfo info = new SqlInfo();
24271                                                                        info.setSql(JSON.toJSONString(queryObject));
24272                                                                        info.setFileName(fileName);
24273                                                                        info.setFilePath(filePath);
24274                                                                        info.setOriginIndex(0);
24275                                                                        sqlInfos.add(info);
24276                                                                }
24277                                                        }
24278                                                }
24279                                                
24280                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
24281                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
24282                                                        for(Map error: errorMessages){
24283                                                                ErrorInfo errorInfo = new ErrorInfo();
24284                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
24285                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
24286                                                                errorInfo.setFileName(fileName);
24287                                                                errorInfo.setFilePath(filePath);
24288                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
24289                                                                                ModelBindingManager.getGlobalHash()));
24290                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
24291                                                                                ModelBindingManager.getGlobalHash()));
24292                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
24293                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
24294                                                                metadataErrors.add(errorInfo);
24295                                                        }
24296                                                }
24297                                        }
24298                                }
24299                        } else if (sql != null) {
24300                                SqlInfo info = new SqlInfo();
24301                                info.setSql(sql);
24302                                info.setFileName(fileName);
24303                                info.setFilePath(filePath);
24304                                info.setOriginIndex(0);
24305                                sqlInfos.add(info);
24306                        }
24307                }
24308                return sqlInfos;
24309        }
24310
24311        public void setTextFormat(boolean textFormat) {
24312                option.setTextFormat(textFormat);
24313        }
24314
24315        public boolean isBuiltInFunctionName(TObjectName object) {
24316                if (object == null || object.getGsqlparser() == null)
24317                        return false;
24318                try {
24319                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
24320                        if (vendor == EDbVendor.dbvteradata) {
24321                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(object.toString().toUpperCase());
24322                                if (result) {
24323                                        return true;
24324                                }
24325                        }
24326
24327                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
24328                        if (versions != null && versions.size() > 0) {
24329                                for (int i = 0; i < versions.size(); i++) {
24330                                        boolean result = functionChecker.isBuiltInFunction(object.toString(),
24331                                                        object.getGsqlparser().getDbVendor(), versions.get(i));
24332                                        if (result) {
24333                                                return result;
24334                                        }
24335                                }
24336
24337                                // boolean result =
24338                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
24339                                // if (result) {
24340                                // return true;
24341                                // }
24342                        }
24343                } catch (Exception e) {
24344                }
24345
24346                return false;
24347        }
24348        
24349        public boolean isBuiltInFunctionName(String functionName) {
24350                if (functionName == null)
24351                        return false;
24352                try {
24353                        EDbVendor vendor = getOption().getVendor();
24354                        if (vendor == EDbVendor.dbvteradata) {
24355                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(functionName.toUpperCase());
24356                                if (result) {
24357                                        return true;
24358                                }
24359                        }
24360
24361                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
24362                        if (versions != null && versions.size() > 0) {
24363                                for (int i = 0; i < versions.size(); i++) {
24364                                        boolean result = functionChecker.isBuiltInFunction(functionName.toUpperCase(),
24365                                                        vendor, versions.get(i));
24366                                        if (result) {
24367                                                return result;
24368                                        }
24369                                }
24370
24371                                // boolean result =
24372                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
24373                                // if (result) {
24374                                // return true;
24375                                // }
24376                        }
24377                } catch (Exception e) {
24378                }
24379
24380                return false;
24381        }
24382
24383        public boolean isKeyword(TObjectName object) {
24384                if (object == null || object.getGsqlparser() == null)
24385                        return false;
24386                try {
24387                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
24388
24389                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
24390                        if (versions != null && versions.size() > 0) {
24391                                for (int i = 0; i < versions.size(); i++) {
24392                                        List<String> segments = SQLUtil.parseNames(object.toString());
24393                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
24394                                                        object.getGsqlparser().getDbVendor(), versions.get(i), true);
24395                                        if (result) {
24396                                                return result;
24397                                        }
24398                                }
24399                        }
24400                } catch (Exception e) {
24401                }
24402
24403                return false;
24404        }
24405        
24406        public boolean isKeyword(String objectName) {
24407                if (objectName == null)
24408                        return false;
24409                try {
24410                        EDbVendor vendor = getOption().getVendor();
24411
24412                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
24413                        if (versions != null && versions.size() > 0) {
24414                                for (int i = 0; i < versions.size(); i++) {
24415                                        List<String> segments = SQLUtil.parseNames(objectName);
24416                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
24417                                                        vendor, versions.get(i), false);
24418                                        if (result) {
24419                                                return result;
24420                                        }
24421                                }
24422                        }
24423                } catch (Exception e) {
24424                }
24425
24426                return false;
24427        }
24428
24429        public boolean isAggregateFunction(TFunctionCall func) {
24430                if (func == null)
24431                        return false;
24432                return Arrays
24433                                .asList(new String[] { "AVG", "COUNT", "MAX", "MIN", "SUM", "COLLECT", "CORR", "COVAR_POP",
24434                                                "COVAR_SAMP", "CUME_DIST", "DENSE_RANK", "FIRST", "GROUP_ID", "GROUPING", "GROUPING_ID", "LAST",
24435                                                "LISTAGG", "MEDIAN", "PERCENT_RANK", "PERCENTILE_CONT", "PERCENTILE_DISC", "RANK",
24436                                                "STATS_BINOMIAL_TEST", "STATS_CROSSTAB", "STATS_F_TEST", "STATS_KS_TEST", "STATS_MODE",
24437                                                "STATS_MW_TEST", "STATS_ONE_WAY_ANOVA", "STATS_WSR_TEST", "STDDEV", "STDDEV_POP", "STDDEV_SAMP",
24438                                                "SYS_XMLAGG", "VAR_ POP", "VAR_ SAMP", "VARI ANCE", "XMLAGG", "ARRAY_AGG" })
24439                                .contains(func.getFunctionName().toString().toUpperCase());
24440        }
24441
24442        public boolean isConstant(TObjectName object) {
24443                if (object == null || object.getGsqlparser() == null)
24444                        return false;
24445                List<String> constants = Arrays.asList(new String[] { "NEXTVAL", "CURRVAL", "SYSDATE", "CENTURY", "YEAR",
24446                                "MONTH", "DAY", "HOUR", "MINUTE", "SECOND" });
24447                List<String> segments = SQLUtil.parseNames(object.toString());
24448                // sequence.NEXTVAL or sequence.CURRVAL is a sequence reference, not a constant
24449                // This syntax is used by Oracle, Snowflake, and accepted by other vendors for compatibility
24450                String columnNameOnly = object.getColumnNameOnly();
24451                if (segments.size() > 1 && ("NEXTVAL".equalsIgnoreCase(columnNameOnly) || "CURRVAL".equalsIgnoreCase(columnNameOnly))) {
24452                        return false;
24453                }
24454                boolean result = constants.indexOf(segments.get(segments.size() - 1).toUpperCase()) != -1;
24455                if (result) {
24456                        return result;
24457                }
24458                if (isKeyword(object)) {
24459                        return true;
24460                }
24461                return false;
24462        }
24463
24464        private Pair3<Long, Long, Integer> convertCoordinate(Pair3<Long, Long, String> position) {
24465//              if (ModelBindingManager.getGlobalOption()!=null && ModelBindingManager.getGlobalOption().isIgnoreCoordinate()) {
24466//                      return new Pair3<>(-1L, -1L, -1);
24467//              }
24468                return new Pair3<Long, Long, Integer>(position.first, position.second,
24469                                ModelBindingManager.getGlobalSqlInfo().getIndexOf(position.third));
24470        }
24471
24472        /**
24473         * Analyze MDX SELECT statement to generate data lineage.
24474         * Maps MDX cube as source table, measures/dimensions as columns,
24475         * and creates dataflow relationships to the result set.
24476         */
24477        private void analyzeMdxSelectStmt(gudusoft.gsqlparser.stmt.mdx.TMdxSelect stmt) {
24478                // Get cube name from FROM clause
24479                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode cube = stmt.getCube();
24480                if (cube == null) {
24481                        return;
24482                }
24483
24484                String cubeName = getMdxIdentifierName(cube);
24485                Table cubeTable = modelFactory.createTableByName(cubeName, false);
24486
24487                // Collect all MDX identifier references from axes and WHERE clause
24488                List<String> measureNames = new ArrayList<String>();
24489                List<String> dimensionNames = new ArrayList<String>();
24490
24491                // Process axes (COLUMNS, ROWS, etc.)
24492                if (stmt.getAxes() != null) {
24493                        for (int i = 0; i < stmt.getAxes().size(); i++) {
24494                                gudusoft.gsqlparser.nodes.mdx.TMdxAxisNode axis = stmt.getAxes().getElement(i);
24495                                if (axis.getExpNode() != null) {
24496                                        collectMdxReferences(axis.getExpNode(), measureNames, dimensionNames);
24497                                }
24498                        }
24499                }
24500
24501                // Process WHERE clause (slicer dimension)
24502                if (stmt.getWhere() != null && stmt.getWhere().getFilter() != null) {
24503                        collectMdxReferences(stmt.getWhere().getFilter(), measureNames, dimensionNames);
24504                }
24505
24506                // Process WITH MEMBER definitions
24507                if (stmt.getWiths() != null) {
24508                        for (int i = 0; i < stmt.getWiths().size(); i++) {
24509                                gudusoft.gsqlparser.nodes.mdx.TMdxWithNode withNode = stmt.getWiths().getElement(i);
24510                                if (withNode.getNameNode() != null) {
24511                                        String withName = getMdxIdentifierName(withNode.getNameNode());
24512                                        // Calculated members are treated as derived measures
24513                                        if (withName.toLowerCase().startsWith("[measures].")
24514                                                        || withName.toLowerCase().startsWith("measures.")) {
24515                                                measureNames.add(withName);
24516                                        }
24517                                }
24518                                // Also collect references used in the WITH expression
24519                                if (withNode.getExprNode() != null) {
24520                                        collectMdxReferences(withNode.getExprNode(), measureNames, dimensionNames);
24521                                }
24522                        }
24523                }
24524
24525                // Create columns on the cube table for all referenced measures and dimensions
24526                Set<String> addedColumns = new LinkedHashSet<String>();
24527                for (String measure : measureNames) {
24528                        if (addedColumns.add(measure)) {
24529                                modelFactory.createTableColumn(cubeTable, measure);
24530                        }
24531                }
24532                for (String dimension : dimensionNames) {
24533                        if (addedColumns.add(dimension)) {
24534                                modelFactory.createTableColumn(cubeTable, dimension);
24535                        }
24536                }
24537
24538                // Create result set with all referenced columns
24539                ResultSet resultSet = modelFactory.createResultSet(stmt, false);
24540                if (resultSet != null) {
24541                        for (String colName : addedColumns) {
24542                                TableColumn sourceCol = findTableColumnByName(cubeTable, colName);
24543                                if (sourceCol != null) {
24544                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24545                                        relation.setEffectType(EffectType.select);
24546                                        relation.addSource(new TableColumnRelationshipElement(sourceCol));
24547                                        relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
24548                                                        resultSet.getRelationRows()));
24549                                }
24550                        }
24551                }
24552        }
24553
24554        private TableColumn findTableColumnByName(Table table, String name) {
24555                for (TableColumn col : table.getColumns()) {
24556                        if (col.getName().equals(name)) {
24557                                return col;
24558                        }
24559                }
24560                return null;
24561        }
24562
24563        /**
24564         * Extract the display name from an MDX identifier node.
24565         * E.g., [Measures].[Departures NEAT] -> [Measures].[Departures NEAT]
24566         */
24567        private String getMdxIdentifierName(gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode) {
24568                StringBuilder sb = new StringBuilder();
24569                for (int i = 0; i < idNode.getSegmentList().size(); i++) {
24570                        if (i > 0) sb.append(".");
24571                        gudusoft.gsqlparser.nodes.mdx.IMdxIdentifierSegment seg = idNode.getSegmentList().getElement(i);
24572                        if (seg.getQuoting() == gudusoft.gsqlparser.nodes.mdx.EMdxQuoting.QUOTED) {
24573                                sb.append("[").append(seg.getName()).append("]");
24574                        } else {
24575                                sb.append(seg.getName());
24576                        }
24577                }
24578                return sb.toString();
24579        }
24580
24581        /**
24582         * Recursively collect measure and dimension references from MDX expression tree.
24583         * Uses iterative DFS to avoid StackOverflow on deeply nested expressions.
24584         */
24585        private void collectMdxReferences(gudusoft.gsqlparser.nodes.mdx.TMdxExpNode expr,
24586                        List<String> measures, List<String> dimensions) {
24587                Deque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode> stack = new ArrayDeque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode>();
24588                stack.push(expr);
24589
24590                while (!stack.isEmpty()) {
24591                        gudusoft.gsqlparser.nodes.mdx.TMdxExpNode current = stack.pop();
24592                        if (current == null) continue;
24593
24594                        if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) {
24595                                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode =
24596                                                (gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) current;
24597                                String name = getMdxIdentifierName(idNode);
24598                                if (name.toLowerCase().startsWith("[measures].")
24599                                                || name.toLowerCase().startsWith("measures.")) {
24600                                        measures.add(name);
24601                                } else if (idNode.getSegmentList().size() > 1) {
24602                                        // Multi-segment identifiers that aren't measures are dimensions
24603                                        dimensions.add(name);
24604                                }
24605                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) {
24606                                gudusoft.gsqlparser.nodes.mdx.TMdxSetNode setNode =
24607                                                (gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) current;
24608                                if (setNode.getTupleList() != null) {
24609                                        for (int i = 0; i < setNode.getTupleList().size(); i++) {
24610                                                stack.push(setNode.getTupleList().getElement(i));
24611                                        }
24612                                }
24613                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) {
24614                                gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode tupleNode =
24615                                                (gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) current;
24616                                if (tupleNode.getExprList() != null) {
24617                                        for (int i = 0; i < tupleNode.getExprList().size(); i++) {
24618                                                stack.push(tupleNode.getExprList().getElement(i));
24619                                        }
24620                                }
24621                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) {
24622                                gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode binOp =
24623                                                (gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) current;
24624                                if (binOp.getRightExprNode() != null) stack.push(binOp.getRightExprNode());
24625                                if (binOp.getLeftExprNode() != null) stack.push(binOp.getLeftExprNode());
24626                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) {
24627                                gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode funcNode =
24628                                                (gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) current;
24629                                if (funcNode.getArguments() != null) {
24630                                        for (int i = 0; i < funcNode.getArguments().size(); i++) {
24631                                                stack.push(funcNode.getArguments().getElement(i));
24632                                        }
24633                                }
24634                        }
24635                }
24636        }
24637
24638}